diff --git a/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 b/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 index 133cf64be58..0e1cf0db946 100644 --- a/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 +++ b/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 @@ -868,7 +868,6 @@ jsonFunctionName | JSON_OBJECT | JSON_ARRAY | JSON_ARRAY_LENGTH - | TO_JSON_STRING | JSON_EXTRACT | JSON_KEYS | JSON_VALID diff --git a/build.gradle b/build.gradle index 7d1824e7d0d..68161e30c2a 100644 --- a/build.gradle +++ b/build.gradle @@ -129,6 +129,7 @@ allprojects { } configurations.all { resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.9.10" + resolutionStrategy.force "net.minidev:json-smart:${versions.json_smart}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10" resolutionStrategy.force "net.bytebuddy:byte-buddy:1.14.19" resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" diff --git a/core/build.gradle b/core/build.gradle index 9577363c1b4..33fcb11a6b4 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -63,6 +63,8 @@ dependencies { api 'org.apache.calcite:calcite-linq4j:1.38.0' api project(':common') implementation "com.github.seancfoley:ipaddress:5.4.2" + implementation "com.jayway.jsonpath:json-path:2.9.0" + implementation "com.googlecode.aviator:aviator:5.4.3" annotationProcessor('org.immutables:value:2.8.8') compileOnly('org.immutables:value-annotations:2.8.8') diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index b55470509ad..aa99baee61b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -170,6 +170,13 @@ public enum BuiltinFunctionName { LIKE(FunctionName.of("like")), NOT_LIKE(FunctionName.of("not like")), + /** LAMBDA Functions * */ + ARRAY_FORALL(FunctionName.of("forall")), + ARRAY_EXISTS(FunctionName.of("exists")), + ARRAY_FILTER(FunctionName.of("filter")), + ARRAY_TRANSFORM(FunctionName.of("transform")), + ARRAY_AGGREGATE(FunctionName.of("reduce")), + /** Aggregation Function. */ AVG(FunctionName.of("avg")), SUM(FunctionName.of("sum")), @@ -212,9 +219,21 @@ public enum BuiltinFunctionName { TRIM(FunctionName.of("trim")), UPPER(FunctionName.of("upper")), + /** Array Functions. */ + ARRAY(FunctionName.of("array")), + /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), JSON(FunctionName.of("json")), + JSON_OBJECT(FunctionName.of("json_object")), + JSON_ARRAY(FunctionName.of("json_array")), + JSON_ARRAY_LENGTH(FunctionName.of("json_array_length")), + JSON_EXTRACT(FunctionName.of("json_extract")), + JSON_KEYS(FunctionName.of("json_keys")), + JSON_SET(FunctionName.of("json_set")), + JSON_DELETE(FunctionName.of("json_delete")), + JSON_APPEND(FunctionName.of("json_append")), + JSON_EXTEND(FunctionName.of("json_extend")), /** GEOSPATIAL Functions. */ GEOIP(FunctionName.of("geoip")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java index 72d637fd2ba..79ea58b8608 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java @@ -28,7 +28,6 @@ import org.opensearch.sql.expression.datetime.DateTimeFunctions; import org.opensearch.sql.expression.datetime.IntervalClause; import org.opensearch.sql.expression.ip.IPFunctions; -import org.opensearch.sql.expression.json.JsonFunctions; import org.opensearch.sql.expression.operator.arthmetic.ArithmeticFunctions; import org.opensearch.sql.expression.operator.arthmetic.MathematicalFunctions; import org.opensearch.sql.expression.operator.convert.TypeCastOperators; @@ -84,7 +83,6 @@ public static synchronized BuiltinFunctionRepository getInstance() { SystemFunctions.register(instance); OpenSearchFunctions.register(instance); IPFunctions.register(instance); - JsonFunctions.register(instance); } return instance; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index c1ee14a7d7a..7b454b25b5f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -31,6 +31,14 @@ import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.datetime.DateTimeFunctions; +import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonDeleteFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonExtendFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonExtractFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl; import org.opensearch.sql.expression.function.udf.CryptographicFunction; import org.opensearch.sql.expression.function.udf.GrokFunction; import org.opensearch.sql.expression.function.udf.SpanFunction; @@ -64,6 +72,18 @@ /** Defines functions and operators that are implemented only by PPL */ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { + // Json Functions + public static final SqlOperator JSON = new JsonFunctionImpl().toUDF("JSON"); + public static final SqlOperator JSON_ARRAY_LENGTH = + new JsonArrayLengthFunctionImpl().toUDF("JSON_ARRAY_LENGTH"); + public static final SqlOperator JSON_EXTRACT = + new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT"); + public static final SqlOperator JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS"); + public static final SqlOperator JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET"); + public static final SqlOperator JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE"); + public static final SqlOperator JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND"); + public static final SqlOperator JSON_EXTEND = new JsonExtendFunctionImpl().toUDF("JSON_EXTEND"); + // Math functions public static final SqlOperator SPAN = new SpanFunction().toUDF("SPAN"); public static final SqlOperator E = new EulerFunction().toUDF("E"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 554482ad6cc..5a66bd8d58d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -5,9 +5,176 @@ package org.opensearch.sql.expression.function; +import static org.apache.calcite.sql.SqlJsonConstructorNullClause.NULL_ON_NULL; import static org.apache.calcite.sql.type.SqlTypeFamily.IGNORE; import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.getLegacyTypeName; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.*; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ABS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ACOS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.AND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ASCII; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ASIN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN2; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CBRT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CEIL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CEILING; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CIDRMATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COALESCE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONCAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONCAT_WS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONV; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONVERT_TZ; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CRC32; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATEDIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATETIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_ADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_FORMAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_SUB; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYNAME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYOFMONTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYOFWEEK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYOFYEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_MONTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_WEEK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DEGREES; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.E; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EQUAL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EXP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EXTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FLOOR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FROM_DAYS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FROM_UNIXTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.GET_FORMAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.GREATER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.GTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.HOUR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.HOUR_OF_DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IFNULL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_GROK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_ITEM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_EXTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_REPLACE_2; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_BLANK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_EMPTY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NULL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_PRESENT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_APPEND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_ARRAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_ARRAY_LENGTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_DELETE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTEND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_KEYS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_OBJECT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_SET; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_VALID; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LAST_DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LEFT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LENGTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LESS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LIKE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOCALTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOCALTIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOCATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG10; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG2; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOWER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTRIM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKEDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKETIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MD5; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MICROSECOND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_HOUR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MOD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUSFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTHNAME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH_OF_YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOTEQUAL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOW; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NULLIF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.OR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.PERIOD_ADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.PERIOD_DIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.PI; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.POSITION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.POW; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.POWER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.QUARTER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RADIANS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RAND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REGEXP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REPLACE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REVERSE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RIGHT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ROUND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RTRIM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SECOND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SECOND_OF_MINUTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SEC_TO_TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA1; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA2; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIGN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SPAN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SQRT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STRCMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STR_TO_DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBSTR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBSTRING; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SYSDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMEDIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPDIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_FORMAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_TO_SEC; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_DAYS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_SECONDS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRIM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRUNCATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TYPEOF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UNIX_TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UPPER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEKDAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEKOFYEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK_OF_YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.XOR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.YEARWEEK; import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; @@ -21,6 +188,7 @@ import java.util.StringJoiner; import java.util.function.BiFunction; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; @@ -498,6 +666,31 @@ void populate() { registerOperator(WEEK_OF_YEAR, PPLBuiltinOperators.WEEK); registerOperator(WEEKOFYEAR, PPLBuiltinOperators.WEEK); + // Register Json function + register( + JSON_ARRAY, + ((builder, args) -> + builder.makeCall( + SqlStdOperatorTable.JSON_ARRAY, + Stream.concat(Stream.of(builder.makeFlag(NULL_ON_NULL)), Arrays.stream(args)) + .toArray(RexNode[]::new)))); + register( + JSON_OBJECT, + ((builder, args) -> + builder.makeCall( + SqlStdOperatorTable.JSON_OBJECT, + Stream.concat(Stream.of(builder.makeFlag(NULL_ON_NULL)), Arrays.stream(args)) + .toArray(RexNode[]::new)))); + registerOperator(JSON, PPLBuiltinOperators.JSON); + registerOperator(JSON_ARRAY_LENGTH, PPLBuiltinOperators.JSON_ARRAY_LENGTH); + registerOperator(JSON_EXTRACT, PPLBuiltinOperators.JSON_EXTRACT); + registerOperator(JSON_KEYS, PPLBuiltinOperators.JSON_KEYS); + registerOperator(JSON_VALID, SqlStdOperatorTable.IS_JSON_VALUE); + registerOperator(JSON_SET, PPLBuiltinOperators.JSON_SET); + registerOperator(JSON_DELETE, PPLBuiltinOperators.JSON_DELETE); + registerOperator(JSON_APPEND, PPLBuiltinOperators.JSON_APPEND); + registerOperator(JSON_EXTEND, PPLBuiltinOperators.JSON_EXTEND); + // Register implementation. // Note, make the implementation an individual class if too complex. register( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java new file mode 100644 index 00000000000..dd76a002e06 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -0,0 +1,79 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +public class JsonAppendFunctionImpl extends ImplementorUDF { + public JsonAppendFunctionImpl() { + super(new JsonAppendImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return STRING_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class JsonAppendImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonAppendFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) throws JsonProcessingException { + String jsonStr = (String) args[0]; + List keys = Arrays.asList(args).subList(1, args.length); + if (keys.size() % 2 != 0) { + throw new RuntimeException( + "Json append function needs corresponding path and values, but current get: " + keys); + } + JsonNode root = convertInputToJsonNode(args[0]); + List expands = new ArrayList<>(); + for (int i = 0; i < keys.size(); i += 2) { + List expandedPaths = expandJsonPath(root, convertToJsonPath(keys.get(i).toString())); + for (String expandedPath : expandedPaths) { + expands.add( + expandedPath + + MEANING_LESS_KEY_FOR_APPEND_AND_EXTEND); // We add meaningless Key since calcite + // json_insert can only + // insert when the path point to null see: + // https://github.com/apache/calcite/blob/d96709c4cc7ca962601317d0a70914ad95e306e1/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java#L737 + expands.add(keys.get(i + 1)); + } + } + return JsonFunctions.jsonInsert(jsonStr, expands.toArray()); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java new file mode 100644 index 00000000000..cc12d8de7c5 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.INTEGER_FORCE_NULLABLE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; + +import com.google.gson.JsonSyntaxException; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +public class JsonArrayLengthFunctionImpl extends ImplementorUDF { + public JsonArrayLengthFunctionImpl() { + super(new JsonArrayLengthImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return INTEGER_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.STRING; + } + + public static class JsonArrayLengthImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonArrayLengthFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + assert args.length == 1 : "Json array length only accept one argument"; + String value = (String) args[0]; + try { + List target = gson.fromJson(value, List.class); + return target.size(); + } catch (JsonSyntaxException e) { + return null; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java new file mode 100644 index 00000000000..b3a884a4f17 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.apache.calcite.runtime.JsonFunctions.jsonRemove; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; + +import com.fasterxml.jackson.core.JsonProcessingException; +import java.util.Arrays; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +public class JsonDeleteFunctionImpl extends ImplementorUDF { + public JsonDeleteFunctionImpl() { + super(new JsonDeleteImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return STRING_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class JsonDeleteImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonDeleteFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) throws JsonProcessingException { + List jsonPaths = Arrays.asList(args).subList(1, args.length); + String[] pathSpecs = + jsonPaths.stream() + .map(Object::toString) + .map(JsonUtils::convertToJsonPath) + .toArray(String[]::new); + return jsonRemove(args[0].toString(), pathSpecs); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java new file mode 100644 index 00000000000..dd91f1d95bd --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -0,0 +1,94 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +public class JsonExtendFunctionImpl extends ImplementorUDF { + public JsonExtendFunctionImpl() { + super(new JsonExtendImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return STRING_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class JsonExtendImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonExtendFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) throws JsonProcessingException { + String jsonStr = (String) args[0]; + List keys = Arrays.asList(args).subList(1, args.length); + if (keys.size() % 2 != 0) { + throw new RuntimeException( + "Json append function needs corresponding path and values, but current get: " + keys); + } + JsonNode root = convertInputToJsonNode(args[0]); + List expands = new ArrayList<>(); + for (int i = 0; i < keys.size(); i += 2) { + List expandedPaths = expandJsonPath(root, convertToJsonPath(keys.get(i).toString())); + for (String expandedPath : expandedPaths) { + Object value = keys.get(i + 1); + if (value instanceof List targetValues) { + for (Object targetValue : targetValues) { + expands.add(expandedPath + ".meaninglessKey"); + expands.add(targetValue); + } + } else if (value instanceof String stringValue) { + try { + List targetValues = + gson.fromJson(stringValue, List.class); // We first try to extend it as an array + for (Object targetValue : targetValues) { + expands.add(expandedPath + MEANING_LESS_KEY_FOR_APPEND_AND_EXTEND); + expands.add(targetValue); + } + } catch (Exception e) { + expands.add(expandedPath + MEANING_LESS_KEY_FOR_APPEND_AND_EXTEND); + expands.add(value); + } + } else { + expands.add(expandedPath + MEANING_LESS_KEY_FOR_APPEND_AND_EXTEND); + expands.add(value); + } + } + } + return JsonFunctions.jsonInsert(jsonStr, expands.toArray()); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java new file mode 100644 index 00000000000..b08a2584883 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -0,0 +1,102 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.apache.calcite.sql.SqlJsonQueryEmptyOrErrorBehavior.NULL; +import static org.apache.calcite.sql.SqlJsonQueryWrapperBehavior.WITHOUT_ARRAY; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.SqlJsonValueEmptyOrErrorBehavior; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +public class JsonExtractFunctionImpl extends ImplementorUDF { + public JsonExtractFunctionImpl() { + super(new JsonExtractImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return STRING_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class JsonExtractImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonExtractFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + if (args.length < 2) { + return null; + } + JsonFunctions.StatefulFunction a = new JsonFunctions.StatefulFunction(); + String jsonStr = (String) args[0]; + List jsonPaths = Arrays.asList(args).subList(1, args.length); + List pathSpecs = + jsonPaths.stream().map(Object::toString).map(JsonUtils::convertToJsonPath).toList(); + List results = new ArrayList<>(); + for (String pathSpec : pathSpecs) { + Object queryResult = a.jsonQuery(jsonStr, pathSpec, WITHOUT_ARRAY, NULL, NULL, false); + Object valueResult = + a.jsonValue( + jsonStr, + pathSpec, + SqlJsonValueEmptyOrErrorBehavior.NULL, + null, + SqlJsonValueEmptyOrErrorBehavior.NULL, + null); + results.add(queryResult != null ? queryResult : valueResult); + } + if (jsonPaths.size() == 1) { + return doJsonize(results.getFirst()); + } + return doJsonize(results); + } + + private static boolean isScalarObject(Object obj) { + if (obj instanceof Collection) { + return false; + } else { + return !(obj instanceof Map); + } + } + + private static String doJsonize(Object candidate) { + if (isScalarObject(candidate)) { + return candidate.toString(); + } else { + return JsonFunctions.jsonize(candidate); + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java new file mode 100644 index 00000000000..0379aeecb72 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * json(value) Evaluates whether the input can be parsed as JSON format. Returns the value if valid, + * null otherwise. Argument type: ANY Return type: ANY/NULL + */ +public class JsonFunctionImpl extends ImplementorUDF { + public JsonFunctionImpl() { + super(new JsonImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return ReturnTypes.ARG0_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class JsonImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + assert args.length == 1 : "Json only accept one argument"; + if (JsonFunctions.isJsonValue(args[0].toString())) { + return args[0].toString(); + } + return null; + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java new file mode 100644 index 00000000000..40214ca7556 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; + +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +public class JsonKeysFunctionImpl extends ImplementorUDF { + public JsonKeysFunctionImpl() { + super(new JsonKeysImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return STRING_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class JsonKeysImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonKeysFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + assert args.length == 1 : "Json keys only accept one argument"; + String value = JsonFunctions.jsonKeys(args[0].toString()); + if (value.equals("null")) { + return null; + } + return value; + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java new file mode 100644 index 00000000000..27346b478e4 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +public class JsonSetFunctionImpl extends ImplementorUDF { + public JsonSetFunctionImpl() { + super(new JsonSetImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return STRING_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + + public static class JsonSetImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonSetFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + String jsonStr = (String) args[0]; + List keys = Arrays.asList(args).subList(1, args.length); + JsonNode root = convertInputToJsonNode(args[0]); + List expands = new ArrayList<>(); + for (int i = 0; i < keys.size(); i += 2) { + List expandedPaths = expandJsonPath(root, convertToJsonPath(keys.get(i).toString())); + for (String expandedPath : expandedPaths) { + expands.add(expandedPath); + expands.add(keys.get(i + 1)); + } + } + return JsonFunctions.jsonSet(jsonStr, expands.toArray()); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java new file mode 100644 index 00000000000..da8dc2a2413 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java @@ -0,0 +1,152 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import java.util.ArrayList; +import java.util.List; + +public class JsonUtils { + static ObjectMapper objectMapper = new ObjectMapper(); + public static Gson gson = new Gson(); + public static String MEANING_LESS_KEY_FOR_APPEND_AND_EXTEND = ".meaningless_key"; + + /** + * @param input candidate json path like a.b{}.c{2} + * @return the normalized json path like $.a.b[*].c[2] + */ + public static String convertToJsonPath(String input) { + if (input == null || input.isEmpty()) return "$"; + + StringBuilder sb = new StringBuilder("$."); + int i = 0; + while (i < input.length()) { + char c = input.charAt(i); + + if (c == '{') { + int end = input.indexOf('}', i); + if (end == -1) + throw new IllegalArgumentException("Unmatched { in input when converting json path"); + + String index = input.substring(i + 1, end).trim(); + if (index.isEmpty()) { + sb.append("[*]"); + } else { + sb.append("[").append(index).append("]"); + } + i = end + 1; + } else if (c == '.') { + sb.append("."); + i++; + } else { + int start = i; + while (i < input.length() && input.charAt(i) != '.' && input.charAt(i) != '{') { + i++; + } + sb.append(input, start, i); + } + } + + return sb.toString(); + } + + /** + * Transfer the object input to json node + * + * @param input + * @return + */ + public static JsonNode convertInputToJsonNode(Object input) { + try { + JsonNode root; + if (input instanceof String) { + root = objectMapper.readTree(input.toString()); + } else { + root = objectMapper.valueToTree(input); + } + return root; + } catch (Exception e) { + throw new RuntimeException("fail to parse input", e); + } + } + + /** + * The function will expand the json path to eliminate the wildcard *. For example, a[*] would be + * a[0], a[1]... + * + * @param root The json node + * @param rawPath original path + * @return List of expanded paths + */ + public static List expandJsonPath(JsonNode root, String rawPath) { + // Remove only leading "$." or "$" + String cleanedPath = rawPath.replaceFirst("^\\$\\.", "").replaceFirst("^\\$", ""); + + String[] parts = cleanedPath.split("\\."); + return expand(root, parts, 0, "$"); + } + + private static List expand( + JsonNode currentNode, String[] parts, int index, String prefix) { + if (index >= parts.length || currentNode == null) { + return List.of(prefix); + } + + String part = parts[index]; + List results = new ArrayList<>(); + + if (part.endsWith("[*]")) { // Contains wildcard symbol + String field = part.substring(0, part.length() - 3); + JsonNode arrayNode; + if (field.isEmpty()) { + arrayNode = currentNode; + } else { + arrayNode = currentNode.get(field); + } + if (arrayNode != null && arrayNode.isArray()) { + for (int i = 0; i < arrayNode.size(); i++) { + String newPrefix = prefix + "." + field + "[" + i + "]"; + results.addAll(expand(arrayNode.get(i), parts, index + 1, newPrefix)); + } + } + } else if (part.endsWith("]")) { // Normal index symbol + int leftBracketIndex = part.lastIndexOf('['); + String field = part.substring(0, part.length() - 3); + JsonNode arrayNode; + if (field.isEmpty()) { + arrayNode = currentNode; + } else { + arrayNode = currentNode.get(field); + } + Boolean arrayFlag = false; + if (leftBracketIndex > -1 + && part.substring(leftBracketIndex + 1, part.length() - 1).matches("-?\\d+")) { + int currentIndex = + Integer.parseInt(part.substring(leftBracketIndex + 1, part.length() - 1)); + if (arrayNode != null && arrayNode.isArray()) { + if (arrayNode.size() > currentIndex) { + String newPrefix = prefix + "." + part; + results.addAll(expand(arrayNode.get(currentIndex), parts, index + 1, newPrefix)); + arrayFlag = true; + } + } + } + if (!arrayFlag) { // normal keys + JsonNode next = currentNode.get(part); + String newPrefix = prefix + "." + part; + results.addAll(expand(next, parts, index + 1, newPrefix)); + } + } else { // ends with string + JsonNode next = currentNode.get(part); + String newPrefix = prefix + "." + part; + results.addAll(expand(next, parts, index + 1, newPrefix)); + } + + return results; + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java deleted file mode 100644 index 75f134aa4e9..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.json; - -import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; -import static org.opensearch.sql.expression.function.FunctionDSL.define; -import static org.opensearch.sql.expression.function.FunctionDSL.impl; -import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; - -import lombok.experimental.UtilityClass; -import org.opensearch.sql.expression.function.BuiltinFunctionName; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.DefaultFunctionResolver; -import org.opensearch.sql.utils.JsonUtils; - -@UtilityClass -public class JsonFunctions { - public void register(BuiltinFunctionRepository repository) { - repository.register(jsonValid()); - repository.register(jsonFunction()); - } - - private DefaultFunctionResolver jsonValid() { - return define( - BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); - } - - private DefaultFunctionResolver jsonFunction() { - return define( - BuiltinFunctionName.JSON.getName(), - impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); - } -} diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index bba8475c110..8159fd6c115 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,64 +7,22 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; -import java.util.LinkedHashMap; +import com.fasterxml.jackson.databind.JsonNode; import java.util.List; -import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.LiteralExpression; +import org.opensearch.sql.expression.function.jsonUDF.JsonUtils; @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { - @Test - public void json_valid_returns_false() { - List expressions = - List.of( - DSL.literal(LITERAL_MISSING), // missing returns false - DSL.literal(LITERAL_NULL), // null returns false - DSL.literal("invalid"), // invalid type - DSL.literal("{{[}}"), // missing bracket - DSL.literal("[}"), // missing bracket - DSL.literal("}"), // missing bracket - DSL.literal("\"missing quote"), // missing quote - DSL.literal("abc"), // not a type - DSL.literal("97ab"), // not a type - DSL.literal("{1, 2, 3, 4}"), // invalid object - DSL.literal("{\"invalid\":\"json\", \"string\"}"), // invalid object - DSL.literal("{123: 1, true: 2, null: 3}"), // invalid object - DSL.literal("[\"a\": 1, \"b\": 2]") // invalid array - ); - - expressions.stream() - .forEach( - expr -> - assertEquals( - LITERAL_FALSE, - DSL.jsonValid(expr).valueOf(), - "Expected FALSE when calling jsonValid with " + expr)); - } - @Test public void json_valid_throws_ExpressionEvaluationException() { assertThrows( @@ -72,126 +30,6 @@ public void json_valid_throws_ExpressionEvaluationException() { () -> DSL.jsonValid(DSL.literal((ExprValueUtils.booleanValue(true)))).valueOf()); } - @Test - public void json_valid_returns_true() { - - List validJsonStrings = - List.of( - // test json objects are valid - "{\"a\":\"1\",\"b\":\"2\"}", - "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", - "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", - - // test json arrays are valid - "[1, 2, 3, 4]", - "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", - - // test json scalars are valid - "\"abc\"", - "1234", - "12.34", - "true", - "false", - "null", - - // test empty string is valid - ""); - - validJsonStrings.stream() - .forEach( - str -> - assertEquals( - LITERAL_TRUE, - DSL.jsonValid(DSL.literal(str)).valueOf(), - String.format("String %s must be valid json", str))); - } - - @Test - void json_returnsJsonObject() { - FunctionExpression exp; - - // Setup - final String objectJson = - "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " - + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; - - LinkedHashMap objectMap = new LinkedHashMap<>(); - objectMap.put("foo", new ExprStringValue("foo")); - objectMap.put("fuzz", ExprBooleanValue.of(true)); - objectMap.put("bar", new ExprLongValue(1234)); - objectMap.put("bar2", new ExprDoubleValue(12.34)); - objectMap.put("baz", ExprNullValue.of()); - objectMap.put( - "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); - objectMap.put( - "arr", - new ExprCollectionValue( - List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); - ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); - - // exercise - exp = DSL.stringToJson(DSL.literal(objectJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprTupleValue); - assertEquals(expectedTupleExpr, value); - - // also test the empty object case - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.stringToJson(DSL.literal("{}")).valueOf()); - } - - @Test - void json_returnsJsonArray() { - FunctionExpression exp; - - // Setup - final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; - ExprValue expectedArrayExpr = - new ExprCollectionValue( - List.of( - new ExprStringValue("foo"), - new ExprStringValue("fuzz"), - LITERAL_TRUE, - new ExprStringValue("bar"), - new ExprIntegerValue(1234), - new ExprDoubleValue(12.34), - LITERAL_NULL)); - - // exercise - exp = DSL.stringToJson(DSL.literal(arrayJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprCollectionValue); - assertEquals(expectedArrayExpr, value); - - // also test the empty-array case - assertEquals(new ExprCollectionValue(List.of()), DSL.stringToJson(DSL.literal("[]")).valueOf()); - } - - @Test - void json_returnsScalar() { - assertEquals( - new ExprStringValue("foobar"), DSL.stringToJson(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.stringToJson(DSL.literal("1234")).valueOf()); - - assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); - - assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); - assertEquals(LITERAL_FALSE, DSL.stringToJson(DSL.literal("false")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal(LITERAL_NULL)).valueOf()); - - assertEquals(LITERAL_MISSING, DSL.stringToJson(DSL.literal(LITERAL_MISSING)).valueOf()); - - assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("")).valueOf()); - } - @Test void json_returnsSemanticCheckException() { List expressions = @@ -217,4 +55,57 @@ void json_returnsSemanticCheckException() { () -> DSL.castJson(expr).valueOf(), "Expected to throw SemanticCheckException when calling castJson with " + expr)); } + + @Test + void test_convertToJsonPath() { + List originalJsonPath = List.of("{}", "a.b.c", "a{2}.c", "{3}.bc{}.d{1}"); + List targetJsonPath = List.of("$.[*]", "$.a.b.c", "$.a[2].c", "$.[3].bc[*].d[1]"); + List convertedJsonPath = + originalJsonPath.stream().map(JsonUtils::convertToJsonPath).toList(); + assertEquals(targetJsonPath, convertedJsonPath); + } + + @Test + void test_convertToJsonPathWithWrongPath() { + IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> convertToJsonPath("a.{")); + assertEquals(e.getMessage(), "Unmatched { in input when converting json path"); + } + + @Test + void test_jsonPathExpand() { + String jsonStr = + "{\"a\": {\"b\": {\"c\": 1}}, \"a2\": [{\"b2\": [{\"c2\": 1}, {\"c2\": 2}]}, {\"b2\":" + + " [{\"c2\": 1}, {\"c2\": 2}]}, {\"b2\": [{\"c2\": 1}]}], \"a3\": [{\"b3\": [{\"c2\":" + + " 1}, {\"c2\": 2}]}, {\"b4\": [{\"c2\": 1}, {\"c2\": 2}]}, {\"b5\": [{\"c2\": 1}," + + " {\"c2\": 2}]}]}"; + JsonNode node = convertInputToJsonNode(jsonStr); + String candidate1 = "$.a.b.c"; + List target1 = List.of("$.a.b.c"); + assertEquals(expandJsonPath(node, candidate1), target1); + String candidate2 = "$.a2[*].b2[*].c2"; + List target2 = + List.of( + "$.a2[0].b2[0].c2", + "$.a2[0].b2[1].c2", + "$.a2[1].b2[0].c2", + "$.a2[1].b2[1].c2", + "$.a2[2].b2[0].c2"); + assertEquals(expandJsonPath(node, candidate2), target2); + String candidate3 = "$.a3[*].b3[*].c2"; + List target3 = List.of("$.a3[0].b3[0].c2", "$.a3[0].b3[1].c2"); + assertEquals(expandJsonPath(node, candidate3), target3); + String candidate4 = "$.a2[*].b2[1].c2"; + List target4 = List.of("$.a2[0].b2[1].c2", "$.a2[1].b2[1].c2", "$.a2[2].b2[1]"); + assertEquals(expandJsonPath(node, candidate4), target4); + } + + @Test + void test_jsonPathExpandAtArray() { + String jsonStr = "[{\"c\": 1}, {\"c\": 1}, {\"c\": 1}]"; + JsonNode node = convertInputToJsonNode(jsonStr); + String candidate1 = "$.[*]"; + List target1 = List.of("$.[0]", "$.[1]", "$.[2]"); + assertEquals(expandJsonPath(node, candidate1), target1); + } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 77d9d00f45f..8357c26943a 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -8,32 +8,30 @@ JSON Functions :local: :depth: 1 -JSON_VALID ----------- + + + +JSON Path +--------- Description >>>>>>>>>>> -Usage: `json_valid(json_string)` checks if `json_string` is a valid JSON-encoded string. +All JSON paths used in JSON functions follow the format ``{}.{}...``. -Argument type: STRING +Each ```` represents a field name. The ``{}`` part is optional and is only applicable when the corresponding key refers to an array. -Return type: BOOLEAN +For example:: -Example:: + a{2}.b{0} + +This refers to the element at index 0 of the ``b`` array, which is nested inside the element at index 2 of the ``a`` array. + +Notes: + +1. The ``{}`` notation applies **only when** the associated key points to an array. - > source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid - fetched rows / total rows = 6/6 - +---------------------+---------------------------------+----------+ - | test_name | json_string | is_valid | - |---------------------|---------------------------------|----------| - | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True | - | json object | {"a":"1","b":"2"} | True | - | json array | [1, 2, 3, 4] | True | - | json scalar string | "abc" | True | - | json empty string | | True | - | json invalid object | {"invalid":"json", "string"} | False | - +---------------------+---------------------------------+----------+ +2. ``{}`` (without a specific index) is interpreted as a **wildcard**, equivalent to ``{*}``, meaning "all elements" in the array at that level. JSON ---------- @@ -41,11 +39,15 @@ JSON Description >>>>>>>>>>> -Usage: `json(value)` Evaluates whether a string can be parsed as a json-encoded string and casted as an expression. Returns the JSON value if valid, null otherwise. +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + +Usage: `json(value)` Evaluates whether a string can be parsed as a json-encoded string. Returns the value if valid, null otherwise. Argument type: STRING -Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY +Return type: STRING Example:: @@ -60,3 +62,317 @@ Example:: | json scalar string | "abc" | "abc" | | json empty string | | null | +---------------------+---------------------------------+-------------------------+ + +JSON_OBJECT +---------- + +Description +>>>>>>>>>>> + +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + +Usage: `json_object(key1, value1, key2, value2...)` create a json object string with key value pairs. The key must be string. + +Argument type: key1: STRING, value1: ANY, key2: STRING, value2: ANY ... + +Return type: STRING + +Example:: + + > source=json_test | eval test_json = json_object('key', 123.45) | head 1 | fields test_json + fetched rows / total rows = 1/1 + +-------------------------+ + | test_json | + |-------------------------| + | {"key":123.45} | + +-------------------------+ + +JSON_ARRAY +---------- + +Description +>>>>>>>>>>> + +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + +Usage: `json_array(element1, element2, ...)` create a json array string with elements. + +Argument type: element1: ANY, element2: ANY ... + +Return type: STRING + +Example:: + + > source=json_test | eval test_json_array = json_array('key', 123.45) | head 1 | fields test_json_array + fetched rows / total rows = 1/1 + +-------------------------+ + | test_json_array | + |-------------------------| + | ["key",123.45] | + +-------------------------+ + +JSON_ARRAY_LENGTH +---------- + +Description +>>>>>>>>>>> + +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + +Usage: `json_array_length(value)` parse the string to json array and return size,, null is returned in case of any other valid JSON string, null or an invalid JSON. + +Argument type: value: A JSON STRING + +Return type: INTEGER + +Example:: + + > source=json_test | eval array_length = json_array_length("[1,2,3]") | head 1 | fields array_length + fetched rows / total rows = 1/1 + +-------------------------+ + | array_length | + |-------------------------| + | 3 | + +-------------------------+ + + > source=json_test | eval array_length = json_array_length("{\"1\": 2}") | head 1 | fields array_length + fetched rows / total rows = 1/1 + +-------------------------+ + | array_length | + |-------------------------| + | null | + +-------------------------+ + +JSON_EXTRACT +---------- + +Description +>>>>>>>>>>> + +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + +Usage: `json_extract(json_string, path1, path2, ...)` Extracts values using the specified JSON paths. If only one path is provided, it returns a single value. If multiple paths are provided, it returns a JSON Array in the order of the paths. If one path cannot find value, return null as the result for this path. The path use "{}" to represent index for array, "{}" means "{*}". + +Argument type: json_string: STRING, path1: STRING, path2: STRING ... + +Return type: STRING + +Example:: + + > source=json_test | eval extract = json_extract('{"a": [{"b": 1}, {"b": 2}]}', 'a{}.b') | head 1 | fields extract + fetched rows / total rows = 1/1 + +-------------------------+ + | test_json_array | + |-------------------------| + | [1,2] | + +-------------------------+ + + > source=json_test | eval extract = json_extract('{"a": [{"b": 1}, {"b": 2}]}', 'a{}.b', 'a{}') | head 1 | fields extract + fetched rows / total rows = 1/1 + +---------------------------------+ + | test_json_array | + |---------------------------------| + | [[1,2],[{"b": 1}, {"b": 2}]] | + +---------------------------------+ + +JSON_DELETE +---------- + +Description +>>>>>>>>>>> + +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + +Usage: `json_delete(json_string, path1, path2, ...)` Delete values using the specified JSON paths. Return the json string after deleting. If one path cannot find value, do nothing. + +Argument type: json_string: STRING, path1: STRING, path2: STRING ... + +Return type: STRING + +Example:: + + > source=json_test | eval delete = json_delete('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b') | head 1 | fields delete + fetched rows / total rows = 1/1 + +-------------------------+ + | delete | + |-------------------------| + | {"a": [{},{"b": 1}]} | + +-------------------------+ + + > source=json_test | eval delete = json_delete('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', 'a{1}.b') | head 1 | fields delete + fetched rows / total rows = 1/1 + +-------------------------+ + | delete | + |-------------------------| + | {"a": []} | + +-------------------------+ + + > source=json_test | eval delete = json_delete('{"a": [{"b": 1}, {"b": 2}]}', 'a{2}.b') | head 1 | fields delete + fetched rows / total rows = 1/1 + +------------------------------+ + | delete | + |------------------------------| + | {"a": [{"b": 1}, {"b": 2}]} | + +------------------------------+ + +JSON_SET +---------- + +Description +>>>>>>>>>>> + +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + +Usage: `json_set(json_string, path1, value1, path2, value2...)` Set values to corresponding paths using the specified JSON paths. If one path's parent node is not a json object, skip the path. Return the json string after setting. + +Argument type: json_string: STRING, path1: STRING, value1: ANY, path2: STRING, value2: ANY ... + +Return type: STRING + +Example:: + + > source=json_test | eval jsonSet = json_set('{"a": [{"b": 1}]}', 'a{0}.b', 3) | head 1 | fields jsonSet + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonSet | + |-------------------------| + | {"a": [{"b": 3}]} | + +-------------------------+ + + > source=json_test | eval jsonSet = json_set('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', 3, 'a{1}.b', 4) | head 1 | fields jsonSet + fetched rows / total rows = 1/1 + +-----------------------------+ + | jsonSet | + |-----------------------------| + | {"a": [{"b": 3},{"b": 4}]} | + +-----------------------------+ + +JSON_APPEND +---------- + +Description +>>>>>>>>>>> + +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + +Usage: `json_append(json_string, path1, value1, path2, value2...)` Append values to corresponding paths using the specified JSON paths. If one path's target node is not an array, skip the path. Return the json string after setting. + +Argument type: json_string: STRING, path1: STRING, value1: ANY, path2: STRING, value2: ANY ... + +Return type: STRING + +Example:: + + > source=json_test | eval jsonAppend = json_set('{"a": [{"b": 1}]}', 'a', 3) | head 1 | fields jsonAppend + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonAppend | + |-------------------------| + | {"a": [{"b": 1}, 3]} | + +-------------------------+ + + > source=json_test | eval jsonAppend = json_append('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', 3, 'a{1}.b', 4) | head 1 | fields jsonAppend + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonAppend | + |-------------------------| + | {"a": [{"b": 1}, 3]} | + +-------------------------+ + + > source=json_test | eval jsonAppend = json_append('{"a": [{"b": 1}]}', 'a', '[1,2]', 'a{1}.b', 4) | head 1 | fields jsonAppend + fetched rows / total rows = 1/1 + +----------------------------+ + | jsonAppend | + |----------------------------| + | {"a": [{"b": 1}, "[1,2]"]} | + +----------------------------+ + +JSON_EXTEND +---------- + +Description +>>>>>>>>>>> + +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + +Usage: `json_extend(json_string, path1, value1, path2, value2...)` Extend values to corresponding paths using the specified JSON paths. If one path's target node is not an array, skip the path. The function will try to parse the value as an array. If it can be parsed, extend it to the target array. Otherwise, regard the value a single one. Return the json string after setting. + +Argument type: json_string: STRING, path1: STRING, value1: ANY, path2: STRING, value2: ANY ... + +Return type: STRING + +Example:: + + > source=json_test | eval jsonExtend = json_extend('{"a": [{"b": 1}]}', 'a', 3) | head 1 | fields jsonExtend + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonExtend | + |-------------------------| + | {"a": [{"b": 1}, 3]} | + +-------------------------+ + + > source=json_test | eval jsonExtend = json_extend('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', 3, 'a{1}.b', 4) | head 1 | fields jsonExtend + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonExtend | + |-------------------------| + | {"a": [{"b": 1}, 3]} | + +-------------------------+ + + > source=json_test | eval jsonExtend = json_extend('{"a": [{"b": 1}]}', 'a', '[1,2]') | head 1 | fields jsonExtend + fetched rows / total rows = 1/1 + +----------------------------+ + | jsonExtend | + |----------------------------| + | {"a": [{"b": 1},1,2]} | + +----------------------------+ + +JSON_KEYS +---------- + +Description +>>>>>>>>>>> + +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + +Usage: `json_keys(json_string)` Return the key list of the Json object as a Json array. Otherwise, return null. + +Argument type: json_string: A JSON STRING + +Return type: STRING + +Example:: + + > source=json_test | eval jsonKeys = json_keys('{"a": 1, "b": 2}') | head 1 | fields jsonKeys + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonKeys | + |-------------------------| + | ["a","b"] | + +-------------------------+ + + > source=json_test | eval jsonKeys = json_keys('{"a": {"c": 1}, "b": 2}') | head 1 | fields jsonKeys + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonKeys | + |-------------------------| + | ["a","b"] | + +-------------------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteJsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteJsonFunctionsIT.java index 6e0af4881cf..06493982016 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteJsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteJsonFunctionsIT.java @@ -5,8 +5,10 @@ package org.opensearch.sql.calcite.remote; +import org.junit.Ignore; import org.opensearch.sql.ppl.JsonFunctionsIT; +@Ignore public class CalciteJsonFunctionsIT extends JsonFunctionsIT { @Override public void init() throws Exception { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java new file mode 100644 index 00000000000..99623c39410 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -0,0 +1,371 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.standalone; + +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; +import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.MatcherUtils.*; +import static org.opensearch.sql.util.MatcherUtils.rows; + +import java.io.IOException; +import java.util.List; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +public class CalcitePPLJsonBuiltinFunctionIT extends CalcitePPLIntegTestCase { + @Override + public void init() throws IOException { + super.init(); + loadIndex(Index.STATE_COUNTRY); + loadIndex(Index.STATE_COUNTRY_WITH_NULL); + loadIndex(Index.DATE_FORMATS); + loadIndex(Index.BANK_WITH_NULL_VALUES); + loadIndex(Index.DATE); + loadIndex(Index.PEOPLE2); + loadIndex(Index.BANK); + loadIndex(Index.JSON_TEST); + loadIndex(Index.GAME_OF_THRONES); + } + + @Test + public void testJson() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]')," + + " b=json('{\"invalid\": \"json\"')| fields a,b | head 1", + TEST_INDEX_DATE_FORMATS)); + + verifySchema(actual, schema("a", "string"), schema("b", "string")); + + verifyDataRows(actual, rows("[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]", null)); + } + + @Test + public void testJsonObject() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_object('key', 123.45), b=json_object('outer'," + + " json_object('inner', 123.45))| fields a, b | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string"), schema("b", "string")); + + verifyDataRows(actual, rows("{\"key\":123.45}", "{\"outer\":\"{\\\"inner\\\":123.45}\"}")); + } + + @Test + public void testJsonArray() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_array(1, 2, 0, -1, 1.1, -0.11)| fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows(gson.toJson(List.of(1, 2, 0, -1, 1.1, -0.11)))); + } + + @Test + public void testJsonArrayWithDifferentType() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_array(1, '123', json_object(\"name\", 3))| fields a |" + + " head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows("[1,\"123\",\"{\\\"name\\\":3}\"]")); + } + + @Test + public void testJsonArrayLength() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_array_length('[1,2,3,4]'), b =" + + " json_array_length('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]'), c =" + + " json_array_length('{\"key\": 1}') | fields a,b,c | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "integer"), schema("b", "integer"), schema("c", "integer")); + + verifyDataRows(actual, rows(4, 5, null)); + } + + @Test + public void testJsonExtract() { + String candidate = + "[\n" + + "{\n" + + "\"name\":\"London\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Tower Bridge\",\"length\":801.0},\n" + + "{\"name\":\"Millennium Bridge\",\"length\":1066.0}\n" + + "]\n" + + "},\n" + + "{\n" + + "\"name\":\"Venice\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Rialto Bridge\",\"length\":157.0},\n" + + "{\"type\":\"Bridge of Sighs\",\"length\":36.0},\n" + + "{\"type\":\"Ponte della Paglia\"}\n" + + "]\n" + + "},\n" + + "{\n" + + "\"name\":\"San Francisco\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0},\n" + + "{\"name\":\"Bay Bridge\",\"length\":23556.0}\n" + + "]\n" + + "}\n" + + "]"; + JSONObject actual = + executeQuery( + String.format( + "source=%s | head 1 | eval a = json_extract('%s', '{}'), b= json_extract('%s'," + + " '{2}.Bridges{0}.length'), c=json_extract('%s', '{}.Bridges{}.type')," + + " d=json_extract('%s', '{2}.Bridges{0}')| fields a, b,c, d | head 1", + TEST_INDEX_PEOPLE2, candidate, candidate, candidate, candidate)); + + verifySchema( + actual, + schema("a", "string"), + schema("b", "string"), + schema("c", "string"), + schema("d", "string")); + + verifyDataRows( + actual, + rows( + gson.toJson(gson.fromJson(candidate, List.class)), + "8981.0", + "[\"Bridge of Sighs\",\"Ponte della Paglia\"]", + "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0}")); + } + + @Test + public void testJsonExtractWithMultiplyResult() { + String candidate = + "[\n" + + "{\n" + + "\"name\":\"London\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Tower Bridge\",\"length\":801.0},\n" + + "{\"name\":\"Millennium Bridge\",\"length\":1066.0}\n" + + "]\n" + + "},\n" + + "{\n" + + "\"name\":\"Venice\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Rialto Bridge\",\"length\":157.0},\n" + + "{\"type\":\"Bridge of Sighs\",\"length\":36.0},\n" + + "{\"type\":\"Ponte della Paglia\"}\n" + + "]\n" + + "},\n" + + "{\n" + + "\"name\":\"San Francisco\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0},\n" + + "{\"name\":\"Bay Bridge\",\"length\":23556.0}\n" + + "]\n" + + "}\n" + + "]"; + JSONObject actual = + executeQuery( + String.format( + "source=%s | head 1 | eval c=json_extract('%s', '{}.Bridges{}.type'," + + " '{2}.Bridges{0}.length')| fields c | head 1", + TEST_INDEX_PEOPLE2, candidate, candidate, candidate, candidate)); + + verifySchema(actual, schema("c", "string")); + + verifyDataRows(actual, rows("[[\"Bridge of Sighs\",\"Ponte della Paglia\"],8981.0]")); + } + + @Test + public void testJsonKeys() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =" + + " json_keys('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}'), b" + + " =json_keys('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]') | fields a,b | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string"), schema("b", "string")); + + verifyDataRows(actual, rows("[\"f1\",\"f2\"]", null)); + } + + @Test + public void testJsonValid() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =json_valid('[1,2,3,4]'), b =json_valid('{\"invalid\":" + + " \"json\"') | fields a,b | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "boolean"), schema("b", "boolean")); + + verifyDataRows(actual, rows(true, false)); + } + + @Test + public void testJsonSet() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =json_set('{\"a\":[{\"b\":1},{\"b\":2}]}', 'a{}.b', '3')|" + + " fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows("{\"a\":[{\"b\":\"3\"},{\"b\":\"3\"}]}")); + } + + @Test + public void testJsonSetWithWrongPath() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =json_set('{\"a\":[{\"b\":1},{\"b\":2}]}', 'a{}.b.d', '3')|" + + " fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows("{\"a\":[{\"b\":1},{\"b\":2}]}")); + } + + @Test + public void testJsonSetPartialSet() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =json_set('{\"a\":[{\"b\":1},{\"b\":{\"c\": 2}}]}', 'a{}.b.c'," + + " '3')| fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows("{\"a\":[{\"b\":1},{\"b\":{\"c\":\"3\"}}]}")); + } + + @Test + public void testJsonDelete() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a" + + " =json_delete('{\"account_number\":1,\"balance\":39225,\"age\":32,\"gender\":\"M\"}'," + + " 'age','gender')| fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows("{\"account_number\":1,\"balance\":39225}")); + } + + @Test + public void testJsonDeleteWithNested() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a" + + " =json_delete('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}'," + + " 'f2.f3') | fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows("{\"f1\":\"abc\",\"f2\":{\"f4\":\"b\"}}")); + } + + @Test + public void testJsonDeleteWithNestedNothing() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a" + + " =json_delete('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}'," + + " 'f2.f100') | fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows("{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}")); + } + + @Test + public void testJsonDeleteWithNestedAndArray() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a" + + " =json_delete('{\"teacher\":\"Alice\",\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}','teacher'," + + " 'student{}.rank') | fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows("{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}")); + } + + @Test + public void testJsonAppend() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a" + + " =json_append('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," + + " 'student', json_object(\"name\", \"Tomy\",\"rank\", 5)), b =" + + " json_append('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," + + " 'teacher', 'Tom', 'teacher', 'Walt'),c =" + + " json_append('{\"school\":{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}'," + + " 'school.teacher', json_array(\"Tom\", \"Walt\"))| fields a, b, c | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string"), schema("b", "string"), schema("c", "string")); + + verifyDataRows( + actual, + rows( + "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},\"{\\\"name\\\":\\\"Tomy\\\",\\\"rank\\\":5}\"]}", + "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}", + "{\"school\":{\"teacher\":[\"Alice\",\"[\\\"Tom\\\",\\\"Walt\\\"]\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}")); + } + + @Test + public void testJsonExtend() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =" + + " json_extend('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," + + " 'student', json_object(\"name\", \"Tommy\",\"rank\", 5)), b =" + + " json_extend('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," + + " 'teacher', 'Tom', 'teacher', 'Walt'),c =" + + " json_extend('{\"school\":{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}'," + + " 'school.teacher', json_array(\"Tom\", \"Walt\"))| fields a, b, c | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string"), schema("b", "string"), schema("c", "string")); + + verifyDataRows( + actual, + rows( + "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},\"{\\\"name\\\":\\\"Tommy\\\",\\\"rank\\\":5}\"]}", + "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}", + "{\"school\":{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}")); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 9bd40e66535..fa26072e2d7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -11,13 +11,16 @@ import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; +import com.google.gson.Gson; import java.io.IOException; import java.util.List; import java.util.Map; import org.json.JSONArray; import org.json.JSONObject; +import org.junit.Ignore; import org.junit.jupiter.api.Test; +@Ignore("https://github.com/opensearch-project/sql/issues/3565") public class JsonFunctionsIT extends PPLIntegTestCase { @Override public void init() throws Exception { @@ -71,7 +74,13 @@ public void test_cast_json() throws IOException { "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" + " | fields test_name, casted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + String jsonType; + if (isCalciteEnabled()) { + jsonType = "string"; + } else { + jsonType = "undefined"; + } + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, jsonType)); verifyDataRows( result, rows( @@ -102,19 +111,31 @@ public void test_json() throws IOException { "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields" + " test_name, casted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + String jsonType; + if (isCalciteEnabled()) { + jsonType = "string"; + } else { + jsonType = "undefined"; + } + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, jsonType)); JSONObject firstRow = new JSONObject(Map.of("c", 2)); + Object nestedArray = new JSONArray(List.of(1, 2, 3, Map.of("true", true, "number", 123))); + if (isCalciteEnabled()) { + nestedArray = nestedArray.toString(); + } + Object nestedObject = + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(Boolean.FALSE, 3))); + if (isCalciteEnabled()) { + nestedObject = + new Gson() + .toJson(Map.of("d", List.of(Boolean.FALSE, 3), "a", "1", "b", Map.of("c", "3"))); + } verifyDataRows( result, - rows( - "json nested object", - new JSONObject( - Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(Boolean.FALSE, 3)))), + rows("json nested object", nestedObject), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), - rows( - "json nested array", - new JSONArray(List.of(1, 2, 3, Map.of("true", true, "number", 123)))), + rows("json nested array", nestedArray), rows("json scalar string", "abc"), rows("json scalar int", 1234), rows("json scalar float", 12.34), diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index da29ab0f22a..9bfc33dd869 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -377,6 +377,15 @@ ISBLANK: 'ISBLANK'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; +JSON_OBJECT: 'JSON_OBJECT'; +JSON_ARRAY: 'JSON_ARRAY'; +JSON_ARRAY_LENGTH: 'JSON_ARRAY_LENGTH'; +JSON_EXTRACT: 'JSON_EXTRACT'; +JSON_KEYS: 'JSON_KEYS'; +JSON_SET: 'JSON_SET'; +JSON_DELETE: 'JSON_DELETE'; +JSON_APPEND: 'JSON_APPEND'; +JSON_EXTEND: 'JSON_EXTEND'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index dc511e4ce42..e71ba1d8d21 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -707,6 +707,19 @@ trigonometricFunctionName | TAN ; +jsonFunctionName + : JSON + | JSON_OBJECT + | JSON_ARRAY + | JSON_ARRAY_LENGTH + | JSON_EXTRACT + | JSON_KEYS + | JSON_SET + | JSON_DELETE + | JSON_APPEND + | JSON_EXTEND + ; + cryptographicFunctionName : MD5 | SHA1 @@ -884,10 +897,6 @@ positionFunctionName : POSITION ; -jsonFunctionName - : JSON - ; - // operators comparisonOperator : EQUAL @@ -1055,6 +1064,7 @@ keywordsCanBeId | timespanUnit | SPAN | evalFunctionName + | jsonFunctionName | relevanceArgName | intervalUnit | trendlineType @@ -1068,6 +1078,7 @@ keywordsCanBeId | CASE | ELSE | IN + | ARROW | BETWEEN | EXISTS | SOURCE