From 1437f21a079196f0244eb9b87d686e96bc0e0f10 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 17 Apr 2025 15:02:35 +0800 Subject: [PATCH 01/50] add json functions Signed-off-by: xinyual --- .../calcite/utils/BuiltinFunctionUtils.java | 2 + .../function/BuiltinFunctionName.java | 1 + .../function/PPLBuiltinOperators.java | 4 ++ .../expression/function/PPLFuncImpTable.java | 64 ++----------------- .../function/jsonUDF/JsonFunctionImpl.java | 60 +++++++++++++++++ .../jsonUDF/JsonObjectFunctionImpl.java | 59 +++++++++++++++++ integ-test/build.gradle | 6 +- .../CalcitePPLJsonBuiltinFunctionIT.java | 56 ++++++++++++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 10 +-- 10 files changed, 197 insertions(+), 66 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java index 497b2c75102..0c2f0a9092d 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.*; import com.google.common.collect.ImmutableList; +import com.google.gson.Gson; import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; @@ -102,6 +103,7 @@ public interface BuiltinFunctionUtils { static SqlReturnTypeInference VARCHAR_FORCE_NULLABLE = ReturnTypes.VARCHAR.andThen(SqlTypeTransforms.FORCE_NULLABLE); + static Gson gson = new Gson(); static SqlOperator translate(String op) { String capitalOP = op.toUpperCase(Locale.ROOT); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 432eb09d49d..1c07dc93565 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -208,6 +208,7 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), JSON(FunctionName.of("json")), + JSON_OBJECT(FunctionName.of("json_object")), /** GEOSPATIAL Functions. */ GEOIP(FunctionName.of("geoip")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index e1ff69660b1..e6384e54595 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -16,11 +16,15 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable; import org.apache.calcite.util.BuiltInMethod; +import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonObjectFunctionImpl; /** Defines functions and operators that are implemented only by PPL */ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator SPAN = new SpanFunctionImpl().toUDF("SPAN"); + public static final SqlOperator JSON = new JsonFunctionImpl().toUDF("JSON"); + public static final SqlOperator JSON_OBJECT = new JsonObjectFunctionImpl().toUDF("JSON_OBJECT"); /** * Invoking an implementor registered in {@link RexImpTable}, need to use reflection since they're diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index c5bd3db4665..32d428e5f2a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -8,65 +8,7 @@ import static java.lang.Math.E; import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.getLegacyTypeName; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.ABS; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.ACOS; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADD; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.AND; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.ASCII; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.ASIN; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN2; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.CBRT; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.CEILING; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONCAT; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONCAT_WS; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.COS; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.COT; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.DEGREES; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.EQUAL; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.EXP; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.FLOOR; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.GREATER; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.GTE; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NULL; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.LEFT; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.LENGTH; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.LESS; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.LIKE; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.LN; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG10; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG2; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOWER; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTE; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTRIM; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOTEQUAL; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.OR; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.PI; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.POSITION; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.POW; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.POWER; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.RADIANS; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.RAND; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.REGEXP; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.REVERSE; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.RIGHT; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.ROUND; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.RTRIM; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIGN; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIN; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.SPAN; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.STRCMP; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBSTR; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBSTRING; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTRACT; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRIM; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.TYPEOF; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.UPPER; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.XOR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.*; import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; @@ -272,6 +214,10 @@ void populate() { // Register PPL UDF operator registerOperator(SPAN, PPLBuiltinOperators.SPAN); + // Register Json function + registerOperator(JSON, PPLBuiltinOperators.JSON); + registerOperator(JSON_OBJECT, PPLBuiltinOperators.JSON_OBJECT); + // Register implementation. // Note, make the implementation an individual class if too complex. register( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java new file mode 100644 index 00000000000..43f9bea3550 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; + +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; + +/** + * json(value) Evaluates whether a string can be parsed as JSON format. Returns the string value if + * valid, null otherwise. Argument type: STRING Return type: STRING/NULL + */ +public class JsonFunctionImpl extends ImplementorUDF { + public JsonFunctionImpl() { + super(new JsonImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return VARCHAR_FORCE_NULLABLE; + } + + public static class JsonImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + assert args.length == 1 : "Json only accept one argument"; + String value = (String) args[0]; + try { + JsonParser.parseString(value); + return value; + } catch (JsonSyntaxException e) { + return null; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java new file mode 100644 index 00000000000..c494a57668d --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java @@ -0,0 +1,59 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; + +public class JsonObjectFunctionImpl extends ImplementorUDF { + public JsonObjectFunctionImpl() { + super(new JsonObjectImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + return typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), // key type + typeFactory.createSqlType(SqlTypeName.ANY) // value type + ); + }; + } + + public static class JsonObjectImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonObjectFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + Map map = new HashMap<>(); + for (int i = 0; i < args.length; i += 2) { + map.put(args[i], args[i + 1]); + } + return map; + } +} diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 33f2e120bc2..6be82a71faa 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -436,12 +436,12 @@ integTest { dependsOn ':opensearch-sql-plugin:bundlePlugin' if(getOSFamilyType() != "windows") { - dependsOn startPrometheus - finalizedBy stopPrometheus + //dependsOn startPrometheus + //finalizedBy stopPrometheus } // enable calcite codegen in IT - systemProperty 'calcite.debug', 'false' + systemProperty 'calcite.debug', 'true' systemProperty 'org.codehaus.janino.source_debugging.enable', 'false' systemProperty 'org.codehaus.janino.source_debugging.dir', calciteCodegen diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java new file mode 100644 index 00000000000..4f2c9415901 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.standalone; + +import static org.opensearch.sql.legacy.TestsConstants.*; +import static org.opensearch.sql.util.MatcherUtils.*; +import static org.opensearch.sql.util.MatcherUtils.rows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +public class CalcitePPLJsonBuiltinFunctionIT extends CalcitePPLIntegTestCase { + @Override + public void init() throws IOException { + super.init(); + loadIndex(Index.STATE_COUNTRY); + loadIndex(Index.STATE_COUNTRY_WITH_NULL); + loadIndex(Index.DATE_FORMATS); + loadIndex(Index.BANK_WITH_NULL_VALUES); + loadIndex(Index.DATE); + loadIndex(Index.PEOPLE2); + loadIndex(Index.BANK); + } + + @Test + public void testJson() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]')," + + " b=json('{\"invalid\": \"json\"')| fields a,b | head 1", + TEST_INDEX_DATE_FORMATS)); + + verifySchema(actual, schema("a", "string"), schema("b", "string")); + + verifyDataRows(actual, rows("[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]", null)); + } + + @Test + public void testJsonObject() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_object('key', 123.45), b=json_object('outer'," + + " json_object('inner', 123.45))| fields a, b | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "struct"), schema("b", "struct")); + + verifyDataRows(actual, rows("[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]", null)); + } +} diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 00b5509b292..7444f0d9d1c 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -349,6 +349,7 @@ BETWEEN: 'BETWEEN'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; +JSON_OBJECT: 'JSON_OBJECT'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 8a8343afe9e..0934e939998 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -648,6 +648,11 @@ trigonometricFunctionName | TAN ; +jsonFunctionName + : JSON + | JSON_OBJECT + ; + dateTimeFunctionName : ADDDATE | ADDTIME @@ -815,10 +820,6 @@ positionFunctionName : POSITION ; -jsonFunctionName - : JSON - ; - // operators comparisonOperator : EQUAL @@ -976,6 +977,7 @@ keywordsCanBeId | timespanUnit | SPAN | evalFunctionName + | jsonFunctionName | relevanceArgName | intervalUnit | trendlineType From 54356aff984eb488b9391ecad7a1ed357a7d79ca Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 17 Apr 2025 15:07:10 +0800 Subject: [PATCH 02/50] add new functions Signed-off-by: xinyual --- .../jsonUDF/JsonArrayFunctionImpl.java | 57 +++++++++++++++++++ .../jsonUDF/JsonObjectFunctionImpl.java | 2 +- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java new file mode 100644 index 00000000000..ea068e915e9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; + +import java.util.List; + +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; + +public class JsonArrayFunctionImpl extends ImplementorUDF { + public JsonArrayFunctionImpl() { + super(new JsonArrayImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return VARCHAR_FORCE_NULLABLE; + } + + public static class JsonArrayImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonArrayFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + + String value = (String) args[0]; + try { + JsonParser.parseString(value); + return value; + } catch (JsonSyntaxException e) { + return null; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java index c494a57668d..017c38e8d00 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java @@ -23,7 +23,7 @@ public class JsonObjectFunctionImpl extends ImplementorUDF { public JsonObjectFunctionImpl() { - super(new JsonObjectImplementor(), NullPolicy.ANY); + super(new JsonObjectImplementor(), NullPolicy.ALL); } @Override From 7959906d09dc1361c42d1fa88e1f1ce2979af860 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 17 Apr 2025 20:32:43 +0800 Subject: [PATCH 03/50] add json related function Signed-off-by: xinyual --- build.gradle | 1 + core/build.gradle | 1 + .../function/BuiltinFunctionName.java | 5 + .../function/PPLBuiltinOperators.java | 12 ++ .../expression/function/PPLFuncImpTable.java | 6 + .../jsonUDF/JsonArrayFunctionImpl.java | 59 ++++++++-- .../jsonUDF/JsonArrayLengthFunctionImpl.java | 60 ++++++++++ .../jsonUDF/JsonExtractFunctionImpl.java | 62 +++++++++++ .../jsonUDF/JsonKeysFunctionImpl.java | 71 ++++++++++++ .../function/jsonUDF/JsonSetFunctionImpl.java | 60 ++++++++++ .../jsonUDF/JsonValidFunctionImpl.java | 61 ++++++++++ .../jsonUDF/ToJsonStringFunctionImpl.java | 51 +++++++++ .../CalcitePPLJsonBuiltinFunctionIT.java | 105 ++++++++++++++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 5 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 5 + 15 files changed, 552 insertions(+), 12 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java diff --git a/build.gradle b/build.gradle index b3f01bff8d1..2d0473a0518 100644 --- a/build.gradle +++ b/build.gradle @@ -128,6 +128,7 @@ allprojects { } configurations.all { resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.9.10" + resolutionStrategy.force "net.minidev:json-smart:${versions.json_smart}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10" resolutionStrategy.force "net.bytebuddy:byte-buddy:1.14.19" resolutionStrategy.force "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}" diff --git a/core/build.gradle b/core/build.gradle index 9577363c1b4..dca34fcd303 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -63,6 +63,7 @@ dependencies { api 'org.apache.calcite:calcite-linq4j:1.38.0' api project(':common') implementation "com.github.seancfoley:ipaddress:5.4.2" + implementation "com.jayway.jsonpath:json-path:2.9.0" annotationProcessor('org.immutables:value:2.8.8') compileOnly('org.immutables:value-annotations:2.8.8') diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 1c07dc93565..23ba8d4b31c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -209,6 +209,11 @@ public enum BuiltinFunctionName { JSON_VALID(FunctionName.of("json_valid")), JSON(FunctionName.of("json")), JSON_OBJECT(FunctionName.of("json_object")), + JSON_ARRAY(FunctionName.of("json_array")), + TO_JSON_STRING(FunctionName.of("to_json_string")), + JSON_ARRAY_LENGTH(FunctionName.of("json_array_length")), + JSON_EXTRACT(FunctionName.of("json_extract")), + JSON_KEYS(FunctionName.of("json_keys")), /** GEOSPATIAL Functions. */ GEOIP(FunctionName.of("geoip")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index e6384e54595..4c887d49a4d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -16,8 +16,14 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable; import org.apache.calcite.util.BuiltInMethod; +import org.opensearch.sql.expression.function.jsonUDF.JsonArrayFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonExtractFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonObjectFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonValidFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.ToJsonStringFunctionImpl; /** Defines functions and operators that are implemented only by PPL */ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { @@ -25,6 +31,12 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator SPAN = new SpanFunctionImpl().toUDF("SPAN"); public static final SqlOperator JSON = new JsonFunctionImpl().toUDF("JSON"); public static final SqlOperator JSON_OBJECT = new JsonObjectFunctionImpl().toUDF("JSON_OBJECT"); + public static final SqlOperator JSON_ARRAY = new JsonArrayFunctionImpl().toUDF("JSON_ARRAY"); + public static final SqlOperator TO_JSON_STRING = new ToJsonStringFunctionImpl().toUDF("TO_JSON_STRING"); + public static final SqlOperator JSON_ARRAY_LENGTH = new JsonArrayLengthFunctionImpl().toUDF("JSON_ARRAY_LENGTH"); + public static final SqlOperator JSON_EXTRACT = new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT"); + public static final SqlOperator JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS"); + public static final SqlOperator JSON_VALID = new JsonValidFunctionImpl().toUDF("JSON_VALID"); /** * Invoking an implementor registered in {@link RexImpTable}, need to use reflection since they're diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 32d428e5f2a..40b1117ea16 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -217,6 +217,12 @@ void populate() { // Register Json function registerOperator(JSON, PPLBuiltinOperators.JSON); registerOperator(JSON_OBJECT, PPLBuiltinOperators.JSON_OBJECT); + registerOperator(JSON_ARRAY, PPLBuiltinOperators.JSON_ARRAY); + registerOperator(TO_JSON_STRING, PPLBuiltinOperators.TO_JSON_STRING); + registerOperator(JSON_ARRAY_LENGTH, PPLBuiltinOperators.JSON_ARRAY_LENGTH); + registerOperator(JSON_EXTRACT, PPLBuiltinOperators.JSON_EXTRACT); + registerOperator(JSON_KEYS, PPLBuiltinOperators.JSON_KEYS); + registerOperator(JSON_VALID, PPLBuiltinOperators.JSON_VALID); // Register implementation. // Note, make the implementation an individual class if too complex. diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java index ea068e915e9..702ba9de69c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.function.jsonUDF; +import com.fasterxml.jackson.databind.type.TypeFactory; import com.google.gson.JsonParser; import com.google.gson.JsonSyntaxException; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -12,16 +13,38 @@ import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.linq4j.tree.SwitchCase; import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; +/** + * json_array(...) Creates a JSON ARRAY using a list of values. + * + * Argument type: + * + * A can be any kind of value such as string, number, or boolean. + * Return type: ARRAY (Spark ArrayType) + * It will also do implicit convert when we can find common types. + * E.g. json_array(1, 2, 0, -1, 1.1, -0.11) = [1.0,2.0,0.0,-1.0,1.1,-0.11] + * + */ + public class JsonArrayFunctionImpl extends ImplementorUDF { public JsonArrayFunctionImpl() { super(new JsonArrayImplementor(), NullPolicy.ANY); @@ -29,29 +52,41 @@ public JsonArrayFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; + return sqlOperatorBinding -> { + RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); + List argTypes = sqlOperatorBinding.collectOperandTypes(); + RelDataType commonType = typeFactory.leastRestrictive(argTypes); + if (commonType == null) { + throw new IllegalArgumentException("All arguments in json array cannot be converted into one common types"); + } + return createArrayType(typeFactory, typeFactory.createTypeWithNullability(commonType, true), true); + }; } public static class JsonArrayImplementor implements NotNullImplementor { @Override public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(JsonArrayFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + RelDataType realType = call.getType().getComponentType(); + List newArgs = new ArrayList<>(translatedOperands); + assert realType != null; + newArgs.add(Expressions.constant(realType.getSqlTypeName())); + return Expressions.call(Types.lookupMethod( + JsonArrayFunctionImpl.class, "eval", Object[].class), newArgs); } } public static Object eval(Object... args) { + SqlTypeName targetType = (SqlTypeName) args[args.length - 1]; + switch (targetType) { + case DOUBLE: + List unboxed = IntStream.range(0, args.length - 1) + .mapToObj(i -> ((Number) args[i]).doubleValue()).collect(Collectors.toList()); + + return unboxed; - String value = (String) args[0]; - try { - JsonParser.parseString(value); - return value; - } catch (JsonSyntaxException e) { - return null; + default: + return List.of(args); } } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java new file mode 100644 index 00000000000..32e0ba87a64 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; + +import java.util.List; + +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.INTEGER_FORCE_NULLABLE; + +public class JsonArrayLengthFunctionImpl extends ImplementorUDF { + public JsonArrayLengthFunctionImpl() { + super(new JsonArrayLengthImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return INTEGER_FORCE_NULLABLE; + } + + public static class JsonArrayLengthImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonArrayLengthFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + assert args.length == 1 : "Json array length only accept one argument"; + String value = (String) args[0]; + try { + List target = gson.fromJson(value, List.class); + return target.size(); + } catch (JsonSyntaxException e) { + return null; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java new file mode 100644 index 00000000000..e8a8b3cdaa8 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -0,0 +1,62 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; +import com.jayway.jsonpath.JsonPath; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; + +import java.util.List; + +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; + +public class JsonExtractFunctionImpl extends ImplementorUDF { + public JsonExtractFunctionImpl() { + super(new JsonExtractImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return VARCHAR_FORCE_NULLABLE; + } + + public static class JsonExtractImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonExtractFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + if (args.length < 2) { + return null; + } + String value = (String) args[0]; + String path = (String) args[1]; + try { + Object result = JsonPath.read(value, path); + result = result != null ? result.toString() : null; + return result; + } catch (Exception e) { + return null; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java new file mode 100644 index 00000000000..d4fb5b75634 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.BasicSqlType; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; + +import java.lang.reflect.Array; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; + +public class JsonKeysFunctionImpl extends ImplementorUDF { + public JsonKeysFunctionImpl() { + super(new JsonKeysImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return sqlOperatorBinding -> { + RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); + return createArrayType(typeFactory, typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true), true); + }; + } + + public static class JsonKeysImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonKeysFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + assert args.length == 1 : "Json keys only accept one argument"; + String value = (String) args[0]; + try { + Map map = gson.fromJson(value, Map.class); + List demo = Arrays.asList(map.keySet().toArray()); + return Arrays.asList(map.keySet().toArray()); + } catch (JsonSyntaxException e) { + return null; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java new file mode 100644 index 00000000000..f6ad1a96fa9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.jayway.jsonpath.JsonPath; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; + +import java.util.List; + +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; + +public class JsonSetFunctionImpl extends ImplementorUDF { + public JsonSetFunctionImpl() { + super(new JsonSetImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return VARCHAR_FORCE_NULLABLE; + } + + public static class JsonSetImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonExtractFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + if (args.length < 2) { + return null; + } + String value = (String) args[0]; + String path = (String) args[1]; + try { + Object result = JsonPath.read(value, path); + result = result != null ? result.toString() : null; + return result; + } catch (Exception e) { + return null; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java new file mode 100644 index 00000000000..1ffb21d8463 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; + +public class JsonValidFunctionImpl extends ImplementorUDF { + public JsonValidFunctionImpl() { + super(new JsonValidImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return ReturnTypes.BOOLEAN; + } + + public static class JsonValidImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonValidFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + try { + JsonParser.parseString(args[0].toString()); + return true; + } catch (JsonSyntaxException e) { + return false; + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java new file mode 100644 index 00000000000..e66d3f51af9 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java @@ -0,0 +1,51 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; + +import java.util.List; + +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; + +public class ToJsonStringFunctionImpl extends ImplementorUDF { + public ToJsonStringFunctionImpl() { + super(new ToJsonStringImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return VARCHAR_FORCE_NULLABLE; + } + + public static class ToJsonStringImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(ToJsonStringFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + return gson.toJson(args[0]); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 4f2c9415901..3e49849a345 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -10,8 +10,12 @@ import static org.opensearch.sql.util.MatcherUtils.rows; import java.io.IOException; +import java.util.List; + import org.json.JSONObject; +import org.junit.Ignore; import org.junit.jupiter.api.Test; +import org.opensearch.sql.exception.SemanticCheckException; public class CalcitePPLJsonBuiltinFunctionIT extends CalcitePPLIntegTestCase { @Override @@ -53,4 +57,105 @@ public void testJsonObject() { verifyDataRows(actual, rows("[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]", null)); } + + @Test + public void testJsonArray() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_array(1, 2, 0, -1, 1.1, -0.11)| fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "array")); + + verifyDataRows(actual, rows("[1.0, 2.0, 0, -1.0, 1.1, -0.11]")); + } + + @Ignore("We do throw the exception, but current way will resolve safe, so we will get Unsupported operator: json_array") + @Test + public void testJsonArrayWithWrongType() { + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_array(1, 2, 0, true, \"hello\", -0.11)| fields a | head 1", + TEST_INDEX_PEOPLE2)); + }); + verifyErrorMessageContains(e, "unsupported format"); + } + + + @Test + public void testToJsonString() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = to_json_string(json_array(1, 2, 0, -1, 1.1, -0.11))| fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows("[1.0, 2.0, 0.0, -1.0, 1.1, -0.11]")); + } + + @Test + public void testJsonArrayLength() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_array_length('[1,2,3,4]'), b = json_array_length('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]'), c = json_array_length('{\"key\": 1}') | fields a,b,c | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "integer"), schema("b", "integer"), schema("c", "integer")); + + verifyDataRows(actual, rows(4, 5, null)); + } + + @Test + public void testJsonExtract() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_extract('{\"a\":\"b\"}', '$.a'), b = json_extract('{\"a\":[{\"b\":1},{\"b\":2}]}', '$.a[1].b'), c = json_extract('{\"a\":[{\"b\":1},{\"b\":2}]}', '$.a[*].b'), d = json_extract('{\"invalid\": \"json\"') | fields a,b,c,d | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string"), + schema("b", "string"), + schema("c", "string"), + schema("d", "string")); + + verifyDataRows(actual, rows("b", "2", "[1,2]", null)); + } + + @Test + public void testJsonKeys() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_keys('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}'), b =json_keys('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]') | fields a,b | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "array"), + schema("b", "array")); + + verifyDataRows(actual, rows(List.of("f1", "f2"), null)); + } + + @Test + public void testJsonValid() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =json_valid('[1,2,3,4]'), b =json_valid('{\"invalid\": \"json\"') | fields a,b | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "boolean"), + schema("b", "boolean")); + + verifyDataRows(actual, rows(true, false)); + } + } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 7444f0d9d1c..306c80e3eda 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -350,6 +350,11 @@ BETWEEN: 'BETWEEN'; JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; JSON_OBJECT: 'JSON_OBJECT'; +JSON_ARRAY: 'JSON_ARRAY'; +TO_JSON_STRING: 'TO_JSON_STRING'; +JSON_ARRAY_LENGTH: 'JSON_ARRAY_LENGTH'; +JSON_EXTRACT: 'JSON_EXTRACT'; +JSON_KEYS: 'JSON_KEYS'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 0934e939998..6a54f04fa99 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -651,6 +651,11 @@ trigonometricFunctionName jsonFunctionName : JSON | JSON_OBJECT + | JSON_ARRAY + | TO_JSON_STRING + | JSON_ARRAY_LENGTH + | JSON_EXTRACT + | JSON_KEYS ; dateTimeFunctionName From aab67c27b7cea05f9d13aa296908462d98225fe0 Mon Sep 17 00:00:00 2001 From: xinyual Date: Mon, 21 Apr 2025 14:51:15 +0800 Subject: [PATCH 04/50] add all functions Signed-off-by: xinyual --- .../utils/UserDefinedFunctionUtils.java | 2 +- .../function/BuiltinFunctionName.java | 6 + .../function/PPLBuiltinOperators.java | 8 + .../expression/function/PPLFuncImpTable.java | 6 + .../jsonUDF/JsonAppendFunctionImpl.java | 63 +++++++ .../jsonUDF/JsonArrayFunctionImpl.java | 8 +- .../jsonUDF/JsonDeleteFunctionImpl.java | 67 ++++++++ .../jsonUDF/JsonExtendFunctionImpl.java | 63 +++++++ .../jsonUDF/JsonObjectFunctionImpl.java | 4 +- .../function/jsonUDF/JsonSetFunctionImpl.java | 43 +++-- .../function/jsonUDF/JsonUtils.java | 160 ++++++++++++++++++ .../CalcitePPLJsonBuiltinFunctionIT.java | 28 +++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 7 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 10 ++ 14 files changed, 456 insertions(+), 19 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index 176de3474a1..03c97cd11f9 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -104,7 +104,7 @@ public static SqlOperator TransferUserDefinedFunction( udfFunction); } - static SqlReturnTypeInference getReturnTypeInferenceForArray() { + public static SqlReturnTypeInference getReturnTypeInferenceForArray() { return opBinding -> { RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 23ba8d4b31c..59660671d0f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -205,6 +205,9 @@ public enum BuiltinFunctionName { TRIM(FunctionName.of("trim")), UPPER(FunctionName.of("upper")), + /** Array Functions. */ + ARRAY(FunctionName.of("array")), + /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), JSON(FunctionName.of("json")), @@ -214,6 +217,9 @@ public enum BuiltinFunctionName { JSON_ARRAY_LENGTH(FunctionName.of("json_array_length")), JSON_EXTRACT(FunctionName.of("json_extract")), JSON_KEYS(FunctionName.of("json_keys")), + JSON_SET(FunctionName.of("json_set")), + JSON_DELETE(FunctionName.of("json_delete")), + JSON_APPEND(FunctionName.of("json_append")), /** GEOSPATIAL Functions. */ GEOIP(FunctionName.of("geoip")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 4c887d49a4d..8f50cea7480 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -16,12 +16,15 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable; import org.apache.calcite.util.BuiltInMethod; +import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonArrayFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonDeleteFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonExtractFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonObjectFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonValidFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.ToJsonStringFunctionImpl; @@ -37,6 +40,11 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator JSON_EXTRACT = new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT"); public static final SqlOperator JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS"); public static final SqlOperator JSON_VALID = new JsonValidFunctionImpl().toUDF("JSON_VALID"); + public static final SqlOperator ARRAY = new JsonArrayFunctionImpl().toUDF("ARRAY"); + public static final SqlOperator JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET"); + public static final SqlOperator JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE"); + public static final SqlOperator JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND"); + public static final SqlOperator JSON_EXTEND = new JsonExtractFunctionImpl().toUDF("JSON_EXTEND"); /** * Invoking an implementor registered in {@link RexImpTable}, need to use reflection since they're diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 40b1117ea16..0058d17e973 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -214,6 +214,9 @@ void populate() { // Register PPL UDF operator registerOperator(SPAN, PPLBuiltinOperators.SPAN); + // Register Array Function + registerOperator(ARRAY, PPLBuiltinOperators.ARRAY); + // Register Json function registerOperator(JSON, PPLBuiltinOperators.JSON); registerOperator(JSON_OBJECT, PPLBuiltinOperators.JSON_OBJECT); @@ -223,6 +226,9 @@ void populate() { registerOperator(JSON_EXTRACT, PPLBuiltinOperators.JSON_EXTRACT); registerOperator(JSON_KEYS, PPLBuiltinOperators.JSON_KEYS); registerOperator(JSON_VALID, PPLBuiltinOperators.JSON_VALID); + registerOperator(JSON_SET, PPLBuiltinOperators.JSON_SET); + registerOperator(JSON_DELETE, PPLBuiltinOperators.JSON_DELETE); + registerOperator(JSON_APPEND, PPLBuiltinOperators.JSON_APPEND); // Register implementation. // Note, make the implementation an individual class if too complex. diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java new file mode 100644 index 00000000000..77ab004045f --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; + +import java.util.List; +import java.util.Map; + +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; + +public class JsonAppendFunctionImpl extends ImplementorUDF { + public JsonAppendFunctionImpl() { + super(new JsonAppendImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + return typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY) + ); + }; + } + + public static class JsonAppendImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonDeleteFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) throws JsonProcessingException { + String jsonStr = (String) args[0]; + List elements = (List) args[1]; + String demo = updateNestedJson(jsonStr, elements, JsonUtils::appendObjectValue); + Map result = gson.fromJson(demo, Map.class); + return result; + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java index 702ba9de69c..b5d67416717 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java @@ -26,6 +26,7 @@ import org.opensearch.sql.expression.function.ImplementorUDF; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -84,9 +85,12 @@ public static Object eval(Object... args) { .mapToObj(i -> ((Number) args[i]).doubleValue()).collect(Collectors.toList()); return unboxed; - + case FLOAT: + List unboxedFloat = IntStream.range(0, args.length - 1) + .mapToObj(i -> ((Number) args[i]).floatValue()).collect(Collectors.toList()); + return unboxedFloat; default: - return List.of(args); + return Arrays.asList(args).subList(0, args.length - 1); } } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java new file mode 100644 index 00000000000..151913f37c5 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; + +import java.util.List; +import java.util.Map; + +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; + +public class JsonDeleteFunctionImpl extends ImplementorUDF { + public JsonDeleteFunctionImpl() { + super(new JsonDeleteImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + return typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY) + ); + }; + } + + public static class JsonDeleteImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonDeleteFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) throws JsonProcessingException { + String jsonStr = (String) args[0]; + Map jsonMap = objectMapper.readValue(jsonStr, Map.class); + List keys = (List) args[1]; + for (String key : keys) { + String[] keyParts = key.split("\\."); + removeNestedKey(jsonMap, keyParts, 0); + } + Map result = gson.fromJson(objectMapper.writeValueAsString(jsonMap), Map.class); + return result; + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java new file mode 100644 index 00000000000..57cb6fdc87b --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; +import org.opensearch.sql.expression.function.ImplementorUDF; + +import java.util.List; +import java.util.Map; + +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.updateNestedJson; + +public class JsonExtendFunctionImpl extends ImplementorUDF { + public JsonExtendFunctionImpl() { + super(new JsonExtendImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + return typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY) + ); + }; + } + + public static class JsonExtendImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonDeleteFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) throws JsonProcessingException { + String jsonStr = (String) args[0]; + List elements = (List) args[1]; + String demo = updateNestedJson(jsonStr, elements, JsonUtils::extendObjectValue); + Map result = gson.fromJson(demo, Map.class); + return result; + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java index 017c38e8d00..5e7f2989a5c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java @@ -31,8 +31,8 @@ public SqlReturnTypeInference getReturnTypeInference() { return opBinding -> { RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); return typeFactory.createMapType( - typeFactory.createSqlType(SqlTypeName.VARCHAR), // key type - typeFactory.createSqlType(SqlTypeName.ANY) // value type + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY) ); }; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java index f6ad1a96fa9..2eeb032f703 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -5,21 +5,37 @@ package org.opensearch.sql.expression.function.jsonUDF; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; +import com.jayway.jsonpath.PathNotFoundException; +import com.jayway.jsonpath.spi.json.JacksonJsonProvider; +import com.jayway.jsonpath.spi.json.JsonProvider; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; import java.util.List; +import java.util.Map; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.updateNestedJson; public class JsonSetFunctionImpl extends ImplementorUDF { public JsonSetFunctionImpl() { @@ -28,7 +44,13 @@ public JsonSetFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; + return opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + return typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY) + ); + }; } public static class JsonSetImplementor implements NotNullImplementor { @@ -38,23 +60,16 @@ public Expression implement( ScalarFunctionImpl function = (ScalarFunctionImpl) ScalarFunctionImpl.create( - Types.lookupMethod(JsonExtractFunctionImpl.class, "eval", Object[].class)); + Types.lookupMethod(JsonSetFunctionImpl.class, "eval", Object[].class)); return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); } } public static Object eval(Object... args) { - if (args.length < 2) { - return null; - } - String value = (String) args[0]; - String path = (String) args[1]; - try { - Object result = JsonPath.read(value, path); - result = result != null ? result.toString() : null; - return result; - } catch (Exception e) { - return null; - } + String jsonStr = (String) args[0]; + List elements = (List) args[1]; + String demo = updateNestedJson(jsonStr, elements, (obj, key, value) -> obj.put(key, value)); + Map result = gson.fromJson(demo, Map.class); + return result; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java new file mode 100644 index 00000000000..824261ecdcf --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java @@ -0,0 +1,160 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.jsonUDF; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class JsonUtils { + static ObjectMapper objectMapper = new ObjectMapper(); + + static Object parseValue(String value) { + // Try parsing the value as JSON, fallback to primitive if parsing fails + try { + return objectMapper.readValue(value, Object.class); + } catch (Exception e) { + // Primitive value, return as is + return value; + } + } + + @FunctionalInterface + interface UpdateConsumer { + void apply(Map obj, String key, Object value); + } + + private static void traverseNestedObject(Object currentObj, String[] pathParts, int depth, + Object valueToUpdate, UpdateConsumer updateObjectFunction) { + if (currentObj == null || depth >= pathParts.length) { + return; + } + + if (currentObj instanceof Map) { + Map currentMap = (Map) currentObj; + String currentKey = pathParts[depth]; + + if (depth == pathParts.length - 1) { + updateObjectFunction.apply(currentMap, currentKey, valueToUpdate); + } else { + // Continue traversing + currentMap.computeIfAbsent(currentKey, + k -> new LinkedHashMap<>()); // Create map if not present + traverseNestedObject(currentMap.get(currentKey), pathParts, depth + 1, + valueToUpdate, updateObjectFunction); + } + } else if (currentObj instanceof List) { + // If the current object is a list, process each map in the list + List list = (List) currentObj; + for (Object item : list) { + if (item instanceof Map) { + traverseNestedObject(item, pathParts, depth, valueToUpdate, updateObjectFunction); + } + } + } + } + + static String updateNestedJson(String jsonStr, List pathValues, UpdateConsumer updateFieldConsumer) { + if (jsonStr == null) { + return null; + } + // don't update if the list is empty, or the list is not key-value pairs + if (pathValues.isEmpty()) { + return jsonStr; + } + try { + // Parse the JSON string into a Map + Map jsonMap = objectMapper.readValue(jsonStr, Map.class); + + // Iterate through the key-value pairs and update the json + var iter = pathValues.iterator(); + while (iter.hasNext()) { + String path = iter.next(); + if (!iter.hasNext()) { + // no value provided and cannot update anything + break; + } + String[] pathParts = path.split("\\."); + Object parsedValue = parseValue(iter.next()); + + traverseNestedObject(jsonMap, pathParts, 0, parsedValue, updateFieldConsumer); + } + + // Convert the updated map back to JSON + return objectMapper.writeValueAsString(jsonMap); + } catch (Exception e) { + return null; + } + } + + static void appendObjectValue(Map obj, String key, Object value) { + // If it's the last key, append to the array + obj.computeIfAbsent(key, k -> new ArrayList<>()); // Create list if not present + Object existingValue = obj.get(key); + + if (existingValue instanceof List) { + List list = (List) existingValue; + list.add(value); + } + } + + static void extendObjectValue(Map obj, String key, Object value) { + // If it's the last key, append to the array + obj.computeIfAbsent(key, k -> new ArrayList<>()); // Create list if not present + Object existingValue = obj.get(key); + + if (existingValue instanceof List) { + List existingList = (List) existingValue; + if (value instanceof List) { + existingList.addAll((List) value); + } else { + existingList.add(value); + } + } + } + + /** + * remove nested json object using its keys parts. + * + * @param currentObj + * @param keyParts + * @param depth + */ + static void removeNestedKey(Object currentObj, String[] keyParts, int depth) { + if (currentObj == null || depth >= keyParts.length) { + return; + } + + if (currentObj instanceof Map) { + Map currentMap = (Map) currentObj; + String currentKey = keyParts[depth]; + + if (depth == keyParts.length - 1) { + // If it's the last key, remove it from the map + currentMap.remove(currentKey); + } else { + // If not the last key, continue traversing + if (currentMap.containsKey(currentKey)) { + Object nextObj = currentMap.get(currentKey); + + if (nextObj instanceof List) { + // If the value is a list, process each item in the list + List list = (List) nextObj; + for (int i = 0; i < list.size(); i++) { + removeNestedKey(list.get(i), keyParts, depth + 1); + } + } else { + // Continue traversing if it's a map + removeNestedKey(nextObj, keyParts, depth + 1); + } + } + } + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 3e49849a345..ca83b8f54bc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -5,12 +5,14 @@ package org.opensearch.sql.calcite.standalone; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; import static org.opensearch.sql.legacy.TestsConstants.*; import static org.opensearch.sql.util.MatcherUtils.*; import static org.opensearch.sql.util.MatcherUtils.rows; import java.io.IOException; import java.util.List; +import java.util.Map; import org.json.JSONObject; import org.junit.Ignore; @@ -158,4 +160,30 @@ public void testJsonValid() { verifyDataRows(actual, rows(true, false)); } + @Test + public void testArray() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =array(1, 2, 0, -1, 1.1, -0.11) | fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "array")); + + verifyDataRows(actual, rows(List.of(1.0, 2.0, 0, -1.0, 1.1, -0.11))); + } + + @Test + public void testJsonSet() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =json_set('{\"a\":[{\"b\":1},{\"b\":2}]}', array('$.a[*].b', '3', '$.a', '{\"c\":4}'))| fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "struct")); + + verifyDataRows(actual, rows(gson.fromJson("{\"a\":[{\"b\":3},{\"b\":3},{\"c\":4}]}", Map.class))); + } + } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 306c80e3eda..f1355b27b1c 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -346,6 +346,9 @@ ISNOTNULL: 'ISNOTNULL'; CIDRMATCH: 'CIDRMATCH'; BETWEEN: 'BETWEEN'; +// COLLECTION FUNCTIONS +ARRAY: 'ARRAY'; + // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; @@ -355,6 +358,10 @@ TO_JSON_STRING: 'TO_JSON_STRING'; JSON_ARRAY_LENGTH: 'JSON_ARRAY_LENGTH'; JSON_EXTRACT: 'JSON_EXTRACT'; JSON_KEYS: 'JSON_KEYS'; +JSON_SET: 'JSON_SET'; +JSON_DELETE: 'JSON_DELETE'; +JSON_APPEND: 'JSON_APPEND'; +JSON_EXTEND: 'JSON_EXTEND'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 6a54f04fa99..60253dd812d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -530,6 +530,7 @@ evalFunctionName | positionFunctionName | jsonFunctionName | geoipFunctionName + | collectionFunctionName ; functionArgs @@ -648,6 +649,10 @@ trigonometricFunctionName | TAN ; +collectionFunctionName + : ARRAY + ; + jsonFunctionName : JSON | JSON_OBJECT @@ -656,6 +661,10 @@ jsonFunctionName | JSON_ARRAY_LENGTH | JSON_EXTRACT | JSON_KEYS + | JSON_SET + | JSON_DELETE + | JSON_APPEND + | JSON_EXTEND ; dateTimeFunctionName @@ -983,6 +992,7 @@ keywordsCanBeId | SPAN | evalFunctionName | jsonFunctionName + | collectionFunctionName | relevanceArgName | intervalUnit | trendlineType From 8bf3509d6d34cfdbdcc8b934122ddab73fcd3fb4 Mon Sep 17 00:00:00 2001 From: xinyual Date: Mon, 21 Apr 2025 17:17:05 +0800 Subject: [PATCH 05/50] add json functions Signed-off-by: xinyual --- .../function/BuiltinFunctionName.java | 1 + .../function/PPLBuiltinOperators.java | 12 +- .../expression/function/PPLFuncImpTable.java | 1 + .../jsonUDF/JsonAppendFunctionImpl.java | 72 ++--- .../jsonUDF/JsonArrayFunctionImpl.java | 120 ++++---- .../jsonUDF/JsonArrayLengthFunctionImpl.java | 66 +++-- .../jsonUDF/JsonDeleteFunctionImpl.java | 78 +++--- .../jsonUDF/JsonExtendFunctionImpl.java | 72 ++--- .../jsonUDF/JsonExtractFunctionImpl.java | 67 +++-- .../jsonUDF/JsonKeysFunctionImpl.java | 83 +++--- .../jsonUDF/JsonObjectFunctionImpl.java | 3 +- .../function/jsonUDF/JsonSetFunctionImpl.java | 85 +++--- .../function/jsonUDF/JsonUtils.java | 260 +++++++++--------- .../jsonUDF/JsonValidFunctionImpl.java | 60 ++-- .../jsonUDF/ToJsonStringFunctionImpl.java | 51 ++-- integ-test/build.gradle | 4 +- .../CalcitePPLJsonBuiltinFunctionIT.java | 227 +++++++++++---- 17 files changed, 679 insertions(+), 583 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 59660671d0f..6bf99a0d898 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -220,6 +220,7 @@ public enum BuiltinFunctionName { JSON_SET(FunctionName.of("json_set")), JSON_DELETE(FunctionName.of("json_delete")), JSON_APPEND(FunctionName.of("json_append")), + JSON_EXTEND(FunctionName.of("json_extend")), /** GEOSPATIAL Functions. */ GEOIP(FunctionName.of("geoip")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 8f50cea7480..094aa3f9676 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -20,6 +20,7 @@ import org.opensearch.sql.expression.function.jsonUDF.JsonArrayFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonDeleteFunctionImpl; +import org.opensearch.sql.expression.function.jsonUDF.JsonExtendFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonExtractFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl; @@ -35,16 +36,19 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator JSON = new JsonFunctionImpl().toUDF("JSON"); public static final SqlOperator JSON_OBJECT = new JsonObjectFunctionImpl().toUDF("JSON_OBJECT"); public static final SqlOperator JSON_ARRAY = new JsonArrayFunctionImpl().toUDF("JSON_ARRAY"); - public static final SqlOperator TO_JSON_STRING = new ToJsonStringFunctionImpl().toUDF("TO_JSON_STRING"); - public static final SqlOperator JSON_ARRAY_LENGTH = new JsonArrayLengthFunctionImpl().toUDF("JSON_ARRAY_LENGTH"); - public static final SqlOperator JSON_EXTRACT = new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT"); + public static final SqlOperator TO_JSON_STRING = + new ToJsonStringFunctionImpl().toUDF("TO_JSON_STRING"); + public static final SqlOperator JSON_ARRAY_LENGTH = + new JsonArrayLengthFunctionImpl().toUDF("JSON_ARRAY_LENGTH"); + public static final SqlOperator JSON_EXTRACT = + new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT"); public static final SqlOperator JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS"); public static final SqlOperator JSON_VALID = new JsonValidFunctionImpl().toUDF("JSON_VALID"); public static final SqlOperator ARRAY = new JsonArrayFunctionImpl().toUDF("ARRAY"); public static final SqlOperator JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET"); public static final SqlOperator JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE"); public static final SqlOperator JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND"); - public static final SqlOperator JSON_EXTEND = new JsonExtractFunctionImpl().toUDF("JSON_EXTEND"); + public static final SqlOperator JSON_EXTEND = new JsonExtendFunctionImpl().toUDF("JSON_EXTEND"); /** * Invoking an implementor registered in {@link RexImpTable}, need to use reflection since they're diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 0058d17e973..0756347b927 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -229,6 +229,7 @@ void populate() { registerOperator(JSON_SET, PPLBuiltinOperators.JSON_SET); registerOperator(JSON_DELETE, PPLBuiltinOperators.JSON_DELETE); registerOperator(JSON_APPEND, PPLBuiltinOperators.JSON_APPEND); + registerOperator(JSON_EXTEND, PPLBuiltinOperators.JSON_EXTEND); // Register implementation. // Note, make the implementation an individual class if too complex. diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java index 77ab004045f..6ee7c64d783 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -5,7 +5,12 @@ package org.opensearch.sql.expression.function.jsonUDF; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; + import com.fasterxml.jackson.core.JsonProcessingException; +import java.util.List; +import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; @@ -19,45 +24,42 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; -import java.util.List; -import java.util.Map; - -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; - public class JsonAppendFunctionImpl extends ImplementorUDF { - public JsonAppendFunctionImpl() { - super(new JsonAppendImplementor(), NullPolicy.ANY); - } + public JsonAppendFunctionImpl() { + super(new JsonAppendImplementor(), NullPolicy.ANY); + } - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return opBinding -> { - RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); - return typeFactory.createMapType( - typeFactory.createSqlType(SqlTypeName.VARCHAR), - typeFactory.createSqlType(SqlTypeName.ANY) - ); - }; - } + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + return typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY)); + }; + } - public static class JsonAppendImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(JsonDeleteFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } + public static class JsonAppendImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonAppendFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); } + } - public static Object eval(Object... args) throws JsonProcessingException { - String jsonStr = (String) args[0]; - List elements = (List) args[1]; - String demo = updateNestedJson(jsonStr, elements, JsonUtils::appendObjectValue); - Map result = gson.fromJson(demo, Map.class); - return result; + public static Object eval(Object... args) throws JsonProcessingException { + String jsonStr = (String) args[0]; + List keys = (List) args[1]; + if (keys.size() % 2 != 0) { + throw new RuntimeException( + "Json append function needs corresponding path and values, but current get: " + keys); } + String resultStr = updateNestedJson(jsonStr, keys, JsonUtils::appendObjectValue); + Map result = gson.fromJson(resultStr, Map.class); + return result; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java index b5d67416717..08f876db71d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java @@ -5,92 +5,86 @@ package org.opensearch.sql.expression.function.jsonUDF; -import com.fasterxml.jackson.databind.type.TypeFactory; -import com.google.gson.JsonParser; -import com.google.gson.JsonSyntaxException; +import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; -import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; -import org.apache.calcite.linq4j.tree.SwitchCase; import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; - /** * json_array(...) Creates a JSON ARRAY using a list of values. * - * Argument type: - * - * A can be any kind of value such as string, number, or boolean. - * Return type: ARRAY (Spark ArrayType) - * It will also do implicit convert when we can find common types. - * E.g. json_array(1, 2, 0, -1, 1.1, -0.11) = [1.0,2.0,0.0,-1.0,1.1,-0.11] + *

Argument type: * + *

A can be any kind of value such as string, number, or boolean. Return type: ARRAY + * (Spark ArrayType) It will also do implicit convert when we can find common types. E.g. + * json_array(1, 2, 0, -1, 1.1, -0.11) = [1.0,2.0,0.0,-1.0,1.1,-0.11] */ +public class JsonArrayFunctionImpl extends ImplementorUDF { + public JsonArrayFunctionImpl() { + super(new JsonArrayImplementor(), NullPolicy.ANY); + } -public class JsonArrayFunctionImpl extends ImplementorUDF { - public JsonArrayFunctionImpl() { - super(new JsonArrayImplementor(), NullPolicy.ANY); - } + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return sqlOperatorBinding -> { + RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); + List argTypes = sqlOperatorBinding.collectOperandTypes(); + RelDataType commonType = typeFactory.leastRestrictive(argTypes); + if (commonType == null) { + throw new IllegalArgumentException( + "All arguments in json array cannot be converted into one common types"); + } + return createArrayType( + typeFactory, typeFactory.createTypeWithNullability(commonType, true), true); + }; + } + public static class JsonArrayImplementor implements NotNullImplementor { @Override - public SqlReturnTypeInference getReturnTypeInference() { - return sqlOperatorBinding -> { - RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - List argTypes = sqlOperatorBinding.collectOperandTypes(); - RelDataType commonType = typeFactory.leastRestrictive(argTypes); - if (commonType == null) { - throw new IllegalArgumentException("All arguments in json array cannot be converted into one common types"); - } - return createArrayType(typeFactory, typeFactory.createTypeWithNullability(commonType, true), true); - }; - } - - public static class JsonArrayImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - RelDataType realType = call.getType().getComponentType(); - List newArgs = new ArrayList<>(translatedOperands); - assert realType != null; - newArgs.add(Expressions.constant(realType.getSqlTypeName())); - return Expressions.call(Types.lookupMethod( - JsonArrayFunctionImpl.class, "eval", Object[].class), newArgs); - } + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + RelDataType realType = call.getType().getComponentType(); + List newArgs = new ArrayList<>(translatedOperands); + assert realType != null; + newArgs.add(Expressions.constant(realType.getSqlTypeName())); + return Expressions.call( + Types.lookupMethod(JsonArrayFunctionImpl.class, "eval", Object[].class), newArgs); } + } - public static Object eval(Object... args) { - SqlTypeName targetType = (SqlTypeName) args[args.length - 1]; - switch (targetType) { - case DOUBLE: - List unboxed = IntStream.range(0, args.length - 1) - .mapToObj(i -> ((Number) args[i]).doubleValue()).collect(Collectors.toList()); + public static Object eval(Object... args) { + SqlTypeName targetType = (SqlTypeName) args[args.length - 1]; + switch (targetType) { + case DOUBLE: + List unboxed = + IntStream.range(0, args.length - 1) + .mapToObj(i -> ((Number) args[i]).doubleValue()) + .collect(Collectors.toList()); - return unboxed; - case FLOAT: - List unboxedFloat = IntStream.range(0, args.length - 1) - .mapToObj(i -> ((Number) args[i]).floatValue()).collect(Collectors.toList()); - return unboxedFloat; - default: - return Arrays.asList(args).subList(0, args.length - 1); - } + return unboxed; + case FLOAT: + List unboxedFloat = + IntStream.range(0, args.length - 1) + .mapToObj(i -> ((Number) args[i]).floatValue()) + .collect(Collectors.toList()); + return unboxedFloat; + default: + return Arrays.asList(args).subList(0, args.length - 1); } + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java index 32e0ba87a64..ddbcae4de44 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java @@ -5,8 +5,11 @@ package org.opensearch.sql.expression.function.jsonUDF; -import com.google.gson.JsonParser; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.INTEGER_FORCE_NULLABLE; + import com.google.gson.JsonSyntaxException; +import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; @@ -15,46 +18,39 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; -import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; -import java.util.List; - -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; -import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.INTEGER_FORCE_NULLABLE; - public class JsonArrayLengthFunctionImpl extends ImplementorUDF { - public JsonArrayLengthFunctionImpl() { - super(new JsonArrayLengthImplementor(), NullPolicy.ANY); - } + public JsonArrayLengthFunctionImpl() { + super(new JsonArrayLengthImplementor(), NullPolicy.ANY); + } - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return INTEGER_FORCE_NULLABLE; - } + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return INTEGER_FORCE_NULLABLE; + } - public static class JsonArrayLengthImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(JsonArrayLengthFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } + public static class JsonArrayLengthImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonArrayLengthFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); } - - public static Object eval(Object... args) { - assert args.length == 1 : "Json array length only accept one argument"; - String value = (String) args[0]; - try { - List target = gson.fromJson(value, List.class); - return target.size(); - } catch (JsonSyntaxException e) { - return null; - } + } + + public static Object eval(Object... args) { + assert args.length == 1 : "Json array length only accept one argument"; + String value = (String) args[0]; + try { + List target = gson.fromJson(value, List.class); + return target.size(); + } catch (JsonSyntaxException e) { + return null; } + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java index 151913f37c5..24906cc4664 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java @@ -5,7 +5,12 @@ package org.opensearch.sql.expression.function.jsonUDF; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; + import com.fasterxml.jackson.core.JsonProcessingException; +import java.util.List; +import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; @@ -19,49 +24,46 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; -import java.util.List; -import java.util.Map; - -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; - public class JsonDeleteFunctionImpl extends ImplementorUDF { - public JsonDeleteFunctionImpl() { - super(new JsonDeleteImplementor(), NullPolicy.ANY); - } + public JsonDeleteFunctionImpl() { + super(new JsonDeleteImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + return typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY)); + }; + } + public static class JsonDeleteImplementor implements NotNullImplementor { @Override - public SqlReturnTypeInference getReturnTypeInference() { - return opBinding -> { - RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); - return typeFactory.createMapType( - typeFactory.createSqlType(SqlTypeName.VARCHAR), - typeFactory.createSqlType(SqlTypeName.ANY) - ); - }; + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonDeleteFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); } + } - public static class JsonDeleteImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(JsonDeleteFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } + public static Object eval(Object... args) throws JsonProcessingException { + String jsonStr = (String) args[0]; + Map jsonMap = objectMapper.readValue(jsonStr, Map.class); + List keys = (List) args[1]; + if (keys.size() % 2 != 0) { + throw new RuntimeException( + "Json append function needs corresponding path and values, but current get: " + keys); } - - public static Object eval(Object... args) throws JsonProcessingException { - String jsonStr = (String) args[0]; - Map jsonMap = objectMapper.readValue(jsonStr, Map.class); - List keys = (List) args[1]; - for (String key : keys) { - String[] keyParts = key.split("\\."); - removeNestedKey(jsonMap, keyParts, 0); - } - Map result = gson.fromJson(objectMapper.writeValueAsString(jsonMap), Map.class); - return result; + for (String key : keys) { + String[] keyParts = key.split("\\."); + removeNestedKey(jsonMap, keyParts, 0); } + Map result = gson.fromJson(objectMapper.writeValueAsString(jsonMap), Map.class); + return result; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java index 57cb6fdc87b..3b6c282c028 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -5,7 +5,12 @@ package org.opensearch.sql.expression.function.jsonUDF; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.updateNestedJson; + import com.fasterxml.jackson.core.JsonProcessingException; +import java.util.List; +import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; @@ -19,45 +24,42 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; -import java.util.List; -import java.util.Map; - -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.updateNestedJson; - public class JsonExtendFunctionImpl extends ImplementorUDF { - public JsonExtendFunctionImpl() { - super(new JsonExtendImplementor(), NullPolicy.ANY); - } + public JsonExtendFunctionImpl() { + super(new JsonExtendImplementor(), NullPolicy.ANY); + } - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return opBinding -> { - RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); - return typeFactory.createMapType( - typeFactory.createSqlType(SqlTypeName.VARCHAR), - typeFactory.createSqlType(SqlTypeName.ANY) - ); - }; - } + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + return typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY)); + }; + } - public static class JsonExtendImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(JsonDeleteFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } + public static class JsonExtendImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonExtendFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); } + } - public static Object eval(Object... args) throws JsonProcessingException { - String jsonStr = (String) args[0]; - List elements = (List) args[1]; - String demo = updateNestedJson(jsonStr, elements, JsonUtils::extendObjectValue); - Map result = gson.fromJson(demo, Map.class); - return result; + public static Object eval(Object... args) throws JsonProcessingException { + String jsonStr = (String) args[0]; + List keys = (List) args[1]; + if (keys.size() % 2 != 0) { + throw new RuntimeException( + "Json append function needs corresponding path and values, but current get: " + keys); } + String resultStr = updateNestedJson(jsonStr, keys, JsonUtils::extendObjectValue); + Map result = gson.fromJson(resultStr, Map.class); + return result; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index e8a8b3cdaa8..db2ed0e7b45 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -5,9 +5,10 @@ package org.opensearch.sql.expression.function.jsonUDF; -import com.google.gson.JsonParser; -import com.google.gson.JsonSyntaxException; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; + import com.jayway.jsonpath.JsonPath; +import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; @@ -19,44 +20,40 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; -import java.util.List; - -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; - public class JsonExtractFunctionImpl extends ImplementorUDF { - public JsonExtractFunctionImpl() { - super(new JsonExtractImplementor(), NullPolicy.ANY); - } + public JsonExtractFunctionImpl() { + super(new JsonExtractImplementor(), NullPolicy.ANY); + } + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return VARCHAR_FORCE_NULLABLE; + } + + public static class JsonExtractImplementor implements NotNullImplementor { @Override - public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonExtractFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); } + } - public static class JsonExtractImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(JsonExtractFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } + public static Object eval(Object... args) { + if (args.length < 2) { + return null; } - - public static Object eval(Object... args) { - if (args.length < 2) { - return null; - } - String value = (String) args[0]; - String path = (String) args[1]; - try { - Object result = JsonPath.read(value, path); - result = result != null ? result.toString() : null; - return result; - } catch (Exception e) { - return null; - } + String value = (String) args[0]; + String path = (String) args[1]; + try { + Object result = JsonPath.read(value, path); + result = result != null ? result.toString() : null; + return result; + } catch (Exception e) { + return null; } + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java index d4fb5b75634..82959779826 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java @@ -5,67 +5,64 @@ package org.opensearch.sql.expression.function.jsonUDF; -import com.google.gson.JsonParser; +import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; + import com.google.gson.JsonSyntaxException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; -import org.apache.calcite.sql.type.BasicSqlType; -import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; -import java.lang.reflect.Array; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; - public class JsonKeysFunctionImpl extends ImplementorUDF { - public JsonKeysFunctionImpl() { - super(new JsonKeysImplementor(), NullPolicy.ANY); - } + public JsonKeysFunctionImpl() { + super(new JsonKeysImplementor(), NullPolicy.ANY); + } - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return sqlOperatorBinding -> { - RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - return createArrayType(typeFactory, typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true), true); - }; - } + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return sqlOperatorBinding -> { + RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); + return createArrayType( + typeFactory, + typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.VARCHAR), true), + true); + }; + } - public static class JsonKeysImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(JsonKeysFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } + public static class JsonKeysImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonKeysFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); } + } - public static Object eval(Object... args) { - assert args.length == 1 : "Json keys only accept one argument"; - String value = (String) args[0]; - try { - Map map = gson.fromJson(value, Map.class); - List demo = Arrays.asList(map.keySet().toArray()); - return Arrays.asList(map.keySet().toArray()); - } catch (JsonSyntaxException e) { - return null; - } + public static Object eval(Object... args) { + assert args.length == 1 : "Json keys only accept one argument"; + String value = (String) args[0]; + try { + Map map = gson.fromJson(value, Map.class); + List demo = Arrays.asList(map.keySet().toArray()); + return Arrays.asList(map.keySet().toArray()); + } catch (JsonSyntaxException e) { + return null; } + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java index 5e7f2989a5c..4dece7431ef 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java @@ -32,8 +32,7 @@ public SqlReturnTypeInference getReturnTypeInference() { RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); return typeFactory.createMapType( typeFactory.createSqlType(SqlTypeName.VARCHAR), - typeFactory.createSqlType(SqlTypeName.ANY) - ); + typeFactory.createSqlType(SqlTypeName.ANY)); }; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java index 2eeb032f703..ea0fe2c6599 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -5,18 +5,11 @@ package org.opensearch.sql.expression.function.jsonUDF; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.NullNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.jayway.jsonpath.Configuration; -import com.jayway.jsonpath.DocumentContext; -import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.Option; -import com.jayway.jsonpath.PathNotFoundException; -import com.jayway.jsonpath.spi.json.JacksonJsonProvider; -import com.jayway.jsonpath.spi.json.JsonProvider; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.updateNestedJson; + +import java.util.List; +import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; @@ -30,46 +23,42 @@ import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; -import java.util.List; -import java.util.Map; +public class JsonSetFunctionImpl extends ImplementorUDF { + public JsonSetFunctionImpl() { + super(new JsonSetImplementor(), NullPolicy.ANY); + } -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.updateNestedJson; - -public class JsonSetFunctionImpl extends ImplementorUDF { - public JsonSetFunctionImpl() { - super(new JsonSetImplementor(), NullPolicy.ANY); - } + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return opBinding -> { + RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); + return typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.ANY)); + }; + } + public static class JsonSetImplementor implements NotNullImplementor { @Override - public SqlReturnTypeInference getReturnTypeInference() { - return opBinding -> { - RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); - return typeFactory.createMapType( - typeFactory.createSqlType(SqlTypeName.VARCHAR), - typeFactory.createSqlType(SqlTypeName.ANY) - ); - }; - } - - public static class JsonSetImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(JsonSetFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonSetFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); } + } - public static Object eval(Object... args) { - String jsonStr = (String) args[0]; - List elements = (List) args[1]; - String demo = updateNestedJson(jsonStr, elements, (obj, key, value) -> obj.put(key, value)); - Map result = gson.fromJson(demo, Map.class); - return result; + public static Object eval(Object... args) { + String jsonStr = (String) args[0]; + List keys = (List) args[1]; + if (keys.size() % 2 != 0) { + throw new RuntimeException( + "Json append function needs corresponding path and values, but current get: " + keys); } + String resultStr = updateNestedJson(jsonStr, keys, (obj, key, value) -> obj.put(key, value)); + Map result = gson.fromJson(resultStr, Map.class); + return result; + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java index 824261ecdcf..84b6ad1c5f5 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java @@ -6,155 +6,159 @@ package org.opensearch.sql.expression.function.jsonUDF; import com.fasterxml.jackson.databind.ObjectMapper; - import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; public class JsonUtils { - static ObjectMapper objectMapper = new ObjectMapper(); - - static Object parseValue(String value) { - // Try parsing the value as JSON, fallback to primitive if parsing fails - try { - return objectMapper.readValue(value, Object.class); - } catch (Exception e) { - // Primitive value, return as is - return value; - } + static ObjectMapper objectMapper = new ObjectMapper(); + + static Object parseValue(String value) { + // Try parsing the value as JSON, fallback to primitive if parsing fails + try { + return objectMapper.readValue(value, Object.class); + } catch (Exception e) { + // Primitive value, return as is + return value; } - - @FunctionalInterface - interface UpdateConsumer { - void apply(Map obj, String key, Object value); + } + + @FunctionalInterface + interface UpdateConsumer { + void apply(Map obj, String key, Object value); + } + + private static void traverseNestedObject( + Object currentObj, + String[] pathParts, + int depth, + Object valueToUpdate, + UpdateConsumer updateObjectFunction) { + if (currentObj == null || depth >= pathParts.length) { + return; } - private static void traverseNestedObject(Object currentObj, String[] pathParts, int depth, - Object valueToUpdate, UpdateConsumer updateObjectFunction) { - if (currentObj == null || depth >= pathParts.length) { - return; - } - - if (currentObj instanceof Map) { - Map currentMap = (Map) currentObj; - String currentKey = pathParts[depth]; - - if (depth == pathParts.length - 1) { - updateObjectFunction.apply(currentMap, currentKey, valueToUpdate); - } else { - // Continue traversing - currentMap.computeIfAbsent(currentKey, - k -> new LinkedHashMap<>()); // Create map if not present - traverseNestedObject(currentMap.get(currentKey), pathParts, depth + 1, - valueToUpdate, updateObjectFunction); - } - } else if (currentObj instanceof List) { - // If the current object is a list, process each map in the list - List list = (List) currentObj; - for (Object item : list) { - if (item instanceof Map) { - traverseNestedObject(item, pathParts, depth, valueToUpdate, updateObjectFunction); - } - } + if (currentObj instanceof Map) { + Map currentMap = (Map) currentObj; + String currentKey = pathParts[depth]; + + if (depth == pathParts.length - 1) { + updateObjectFunction.apply(currentMap, currentKey, valueToUpdate); + } else { + // Continue traversing + currentMap.computeIfAbsent( + currentKey, k -> new LinkedHashMap<>()); // Create map if not present + traverseNestedObject( + currentMap.get(currentKey), pathParts, depth + 1, valueToUpdate, updateObjectFunction); + } + } else if (currentObj instanceof List) { + // If the current object is a list, process each map in the list + List list = (List) currentObj; + for (Object item : list) { + if (item instanceof Map) { + traverseNestedObject(item, pathParts, depth, valueToUpdate, updateObjectFunction); } + } } + } - static String updateNestedJson(String jsonStr, List pathValues, UpdateConsumer updateFieldConsumer) { - if (jsonStr == null) { - return null; - } - // don't update if the list is empty, or the list is not key-value pairs - if (pathValues.isEmpty()) { - return jsonStr; + static String updateNestedJson( + String jsonStr, List pathValues, UpdateConsumer updateFieldConsumer) { + if (jsonStr == null) { + return null; + } + // don't update if the list is empty, or the list is not key-value pairs + if (pathValues.isEmpty()) { + return jsonStr; + } + try { + // Parse the JSON string into a Map + Map jsonMap = objectMapper.readValue(jsonStr, Map.class); + + // Iterate through the key-value pairs and update the json + var iter = pathValues.iterator(); + while (iter.hasNext()) { + String path = iter.next(); + if (!iter.hasNext()) { + // no value provided and cannot update anything + break; } - try { - // Parse the JSON string into a Map - Map jsonMap = objectMapper.readValue(jsonStr, Map.class); - - // Iterate through the key-value pairs and update the json - var iter = pathValues.iterator(); - while (iter.hasNext()) { - String path = iter.next(); - if (!iter.hasNext()) { - // no value provided and cannot update anything - break; - } - String[] pathParts = path.split("\\."); - Object parsedValue = parseValue(iter.next()); - - traverseNestedObject(jsonMap, pathParts, 0, parsedValue, updateFieldConsumer); - } + String[] pathParts = path.split("\\."); + Object parsedValue = parseValue(iter.next()); - // Convert the updated map back to JSON - return objectMapper.writeValueAsString(jsonMap); - } catch (Exception e) { - return null; - } + traverseNestedObject(jsonMap, pathParts, 0, parsedValue, updateFieldConsumer); + } + + // Convert the updated map back to JSON + return objectMapper.writeValueAsString(jsonMap); + } catch (Exception e) { + return null; } + } - static void appendObjectValue(Map obj, String key, Object value) { - // If it's the last key, append to the array - obj.computeIfAbsent(key, k -> new ArrayList<>()); // Create list if not present - Object existingValue = obj.get(key); + static void appendObjectValue(Map obj, String key, Object value) { + // If it's the last key, append to the array + obj.computeIfAbsent(key, k -> new ArrayList<>()); // Create list if not present + Object existingValue = obj.get(key); - if (existingValue instanceof List) { - List list = (List) existingValue; - list.add(value); - } + if (existingValue instanceof List) { + List list = (List) existingValue; + list.add(value); } - - static void extendObjectValue(Map obj, String key, Object value) { - // If it's the last key, append to the array - obj.computeIfAbsent(key, k -> new ArrayList<>()); // Create list if not present - Object existingValue = obj.get(key); - - if (existingValue instanceof List) { - List existingList = (List) existingValue; - if (value instanceof List) { - existingList.addAll((List) value); - } else { - existingList.add(value); - } - } + } + + static void extendObjectValue(Map obj, String key, Object value) { + // If it's the last key, append to the array + obj.computeIfAbsent(key, k -> new ArrayList<>()); // Create list if not present + Object existingValue = obj.get(key); + + if (existingValue instanceof List) { + List existingList = (List) existingValue; + if (value instanceof List) { + existingList.addAll((List) value); + } else { + existingList.add(value); + } + } + } + + /** + * remove nested json object using its keys parts. + * + * @param currentObj + * @param keyParts + * @param depth + */ + static void removeNestedKey(Object currentObj, String[] keyParts, int depth) { + if (currentObj == null || depth >= keyParts.length) { + return; } - /** - * remove nested json object using its keys parts. - * - * @param currentObj - * @param keyParts - * @param depth - */ - static void removeNestedKey(Object currentObj, String[] keyParts, int depth) { - if (currentObj == null || depth >= keyParts.length) { - return; - } - - if (currentObj instanceof Map) { - Map currentMap = (Map) currentObj; - String currentKey = keyParts[depth]; - - if (depth == keyParts.length - 1) { - // If it's the last key, remove it from the map - currentMap.remove(currentKey); - } else { - // If not the last key, continue traversing - if (currentMap.containsKey(currentKey)) { - Object nextObj = currentMap.get(currentKey); - - if (nextObj instanceof List) { - // If the value is a list, process each item in the list - List list = (List) nextObj; - for (int i = 0; i < list.size(); i++) { - removeNestedKey(list.get(i), keyParts, depth + 1); - } - } else { - // Continue traversing if it's a map - removeNestedKey(nextObj, keyParts, depth + 1); - } - } + if (currentObj instanceof Map) { + Map currentMap = (Map) currentObj; + String currentKey = keyParts[depth]; + + if (depth == keyParts.length - 1) { + // If it's the last key, remove it from the map + currentMap.remove(currentKey); + } else { + // If not the last key, continue traversing + if (currentMap.containsKey(currentKey)) { + Object nextObj = currentMap.get(currentKey); + + if (nextObj instanceof List) { + // If the value is a list, process each item in the list + List list = (List) nextObj; + for (int i = 0; i < list.size(); i++) { + removeNestedKey(list.get(i), keyParts, depth + 1); } + } else { + // Continue traversing if it's a map + removeNestedKey(nextObj, keyParts, depth + 1); + } } + } } + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java index 1ffb21d8463..759333da854 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java @@ -7,55 +7,47 @@ import com.google.gson.JsonParser; import com.google.gson.JsonSyntaxException; +import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; - public class JsonValidFunctionImpl extends ImplementorUDF { - public JsonValidFunctionImpl() { - super(new JsonValidImplementor(), NullPolicy.ANY); - } + public JsonValidFunctionImpl() { + super(new JsonValidImplementor(), NullPolicy.ANY); + } - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return ReturnTypes.BOOLEAN; - } + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return ReturnTypes.BOOLEAN; + } - public static class JsonValidImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(JsonValidFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } + public static class JsonValidImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(JsonValidFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); } - - public static Object eval(Object... args) { - try { - JsonParser.parseString(args[0].toString()); - return true; - } catch (JsonSyntaxException e) { - return false; - } + } + + public static Object eval(Object... args) { + try { + JsonParser.parseString(args[0].toString()); + return true; + } catch (JsonSyntaxException e) { + return false; } + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java index e66d3f51af9..7de40e109ba 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java @@ -5,8 +5,10 @@ package org.opensearch.sql.expression.function.jsonUDF; -import com.google.gson.JsonParser; -import com.google.gson.JsonSyntaxException; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; + +import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; @@ -18,34 +20,29 @@ import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; -import java.util.List; - -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; - public class ToJsonStringFunctionImpl extends ImplementorUDF { - public ToJsonStringFunctionImpl() { - super(new ToJsonStringImplementor(), NullPolicy.ANY); - } + public ToJsonStringFunctionImpl() { + super(new ToJsonStringImplementor(), NullPolicy.ANY); + } - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; - } + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return VARCHAR_FORCE_NULLABLE; + } - public static class ToJsonStringImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(ToJsonStringFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } + public static class ToJsonStringImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(ToJsonStringFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); } + } - public static Object eval(Object... args) { - return gson.toJson(args[0]); - } + public static Object eval(Object... args) { + return gson.toJson(args[0]); + } } diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 6be82a71faa..f661c5a50df 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -436,8 +436,8 @@ integTest { dependsOn ':opensearch-sql-plugin:bundlePlugin' if(getOSFamilyType() != "windows") { - //dependsOn startPrometheus - //finalizedBy stopPrometheus + dependsOn startPrometheus + finalizedBy stopPrometheus } // enable calcite codegen in IT diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index ca83b8f54bc..ae931baebf0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -13,11 +13,9 @@ import java.io.IOException; import java.util.List; import java.util.Map; - import org.json.JSONObject; import org.junit.Ignore; import org.junit.jupiter.api.Test; -import org.opensearch.sql.exception.SemanticCheckException; public class CalcitePPLJsonBuiltinFunctionIT extends CalcitePPLIntegTestCase { @Override @@ -63,40 +61,43 @@ public void testJsonObject() { @Test public void testJsonArray() { JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a = json_array(1, 2, 0, -1, 1.1, -0.11)| fields a | head 1", - TEST_INDEX_PEOPLE2)); + executeQuery( + String.format( + "source=%s | eval a = json_array(1, 2, 0, -1, 1.1, -0.11)| fields a | head 1", + TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "array")); verifyDataRows(actual, rows("[1.0, 2.0, 0, -1.0, 1.1, -0.11]")); } - @Ignore("We do throw the exception, but current way will resolve safe, so we will get Unsupported operator: json_array") + @Ignore( + "We do throw the exception, but current way will resolve safe, so we will get Unsupported" + + " operator: json_array") @Test public void testJsonArrayWithWrongType() { IllegalArgumentException e = - assertThrows( - IllegalArgumentException.class, - () -> { - JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a = json_array(1, 2, 0, true, \"hello\", -0.11)| fields a | head 1", - TEST_INDEX_PEOPLE2)); - }); + assertThrows( + IllegalArgumentException.class, + () -> { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_array(1, 2, 0, true, \"hello\", -0.11)| fields" + + " a | head 1", + TEST_INDEX_PEOPLE2)); + }); verifyErrorMessageContains(e, "unsupported format"); } - @Test public void testToJsonString() { JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a = to_json_string(json_array(1, 2, 0, -1, 1.1, -0.11))| fields a | head 1", - TEST_INDEX_PEOPLE2)); + executeQuery( + String.format( + "source=%s | eval a = to_json_string(json_array(1, 2, 0, -1, 1.1, -0.11))| fields a" + + " | head 1", + TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "string")); @@ -106,10 +107,12 @@ public void testToJsonString() { @Test public void testJsonArrayLength() { JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a = json_array_length('[1,2,3,4]'), b = json_array_length('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]'), c = json_array_length('{\"key\": 1}') | fields a,b,c | head 1", - TEST_INDEX_PEOPLE2)); + executeQuery( + String.format( + "source=%s | eval a = json_array_length('[1,2,3,4]'), b =" + + " json_array_length('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]'), c =" + + " json_array_length('{\"key\": 1}') | fields a,b,c | head 1", + TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "integer"), schema("b", "integer"), schema("c", "integer")); @@ -119,15 +122,20 @@ public void testJsonArrayLength() { @Test public void testJsonExtract() { JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a = json_extract('{\"a\":\"b\"}', '$.a'), b = json_extract('{\"a\":[{\"b\":1},{\"b\":2}]}', '$.a[1].b'), c = json_extract('{\"a\":[{\"b\":1},{\"b\":2}]}', '$.a[*].b'), d = json_extract('{\"invalid\": \"json\"') | fields a,b,c,d | head 1", - TEST_INDEX_PEOPLE2)); + executeQuery( + String.format( + "source=%s | eval a = json_extract('{\"a\":\"b\"}', '$.a'), b =" + + " json_extract('{\"a\":[{\"b\":1},{\"b\":2}]}', '$.a[1].b'), c =" + + " json_extract('{\"a\":[{\"b\":1},{\"b\":2}]}', '$.a[*].b'), d =" + + " json_extract('{\"invalid\": \"json\"') | fields a,b,c,d | head 1", + TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "string"), - schema("b", "string"), - schema("c", "string"), - schema("d", "string")); + verifySchema( + actual, + schema("a", "string"), + schema("b", "string"), + schema("c", "string"), + schema("d", "string")); verifyDataRows(actual, rows("b", "2", "[1,2]", null)); } @@ -135,13 +143,14 @@ public void testJsonExtract() { @Test public void testJsonKeys() { JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a = json_keys('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}'), b =json_keys('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]') | fields a,b | head 1", - TEST_INDEX_PEOPLE2)); + executeQuery( + String.format( + "source=%s | eval a =" + + " json_keys('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}'), b" + + " =json_keys('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]') | fields a,b | head 1", + TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "array"), - schema("b", "array")); + verifySchema(actual, schema("a", "array"), schema("b", "array")); verifyDataRows(actual, rows(List.of("f1", "f2"), null)); } @@ -149,13 +158,13 @@ public void testJsonKeys() { @Test public void testJsonValid() { JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a =json_valid('[1,2,3,4]'), b =json_valid('{\"invalid\": \"json\"') | fields a,b | head 1", - TEST_INDEX_PEOPLE2)); + executeQuery( + String.format( + "source=%s | eval a =json_valid('[1,2,3,4]'), b =json_valid('{\"invalid\":" + + " \"json\"') | fields a,b | head 1", + TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "boolean"), - schema("b", "boolean")); + verifySchema(actual, schema("a", "boolean"), schema("b", "boolean")); verifyDataRows(actual, rows(true, false)); } @@ -163,10 +172,10 @@ public void testJsonValid() { @Test public void testArray() { JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a =array(1, 2, 0, -1, 1.1, -0.11) | fields a | head 1", - TEST_INDEX_PEOPLE2)); + executeQuery( + String.format( + "source=%s | eval a =array(1, 2, 0, -1, 1.1, -0.11) | fields a | head 1", + TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "array")); @@ -176,14 +185,124 @@ public void testArray() { @Test public void testJsonSet() { JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a =json_set('{\"a\":[{\"b\":1},{\"b\":2}]}', array('$.a[*].b', '3', '$.a', '{\"c\":4}'))| fields a | head 1", - TEST_INDEX_PEOPLE2)); + executeQuery( + String.format( + "source=%s | eval a =json_set('{\"a\":[{\"b\":1},{\"b\":2}]}', array('a.b', '3'))|" + + " fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "struct")); + + verifyDataRows(actual, rows(gson.fromJson("{\"a\":[{\"b\":3},{\"b\":3}]}", Map.class))); + } + + @Test + public void testJsonDelete() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a" + + " =json_delete('{\"account_number\":1,\"balance\":39225,\"age\":32,\"gender\":\"M\"}'," + + " array('age','gender'))| fields a | head 1", + TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "struct")); - verifyDataRows(actual, rows(gson.fromJson("{\"a\":[{\"b\":3},{\"b\":3},{\"c\":4}]}", Map.class))); + verifyDataRows( + actual, rows(gson.fromJson("{\"account_number\":1,\"balance\":39225}", Map.class))); } + @Test + public void testJsonDeleteWithNested() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a" + + " =json_delete('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}'," + + " array('f2.f3')) | fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "struct")); + + verifyDataRows( + actual, rows(gson.fromJson("{\"f1\":\"abc\",\"f2\":{\"f4\":\"b\"}}", Map.class))); + } + + @Test + public void testJsonDeleteWithNestedAndArry() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a" + + " =json_delete('{\"teacher\":\"Alice\",\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}',array('teacher'," + + " 'student.rank')) | fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "struct")); + + verifyDataRows( + actual, + rows( + gson.fromJson("{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}", Map.class))); + } + + @Test + public void testJsonAppend() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a" + + " =json_append('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," + + " array('student', '{\"name\":\"Tomy\",\"rank\":5}')), b =" + + " json_append('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," + + " array('teacher', '\"Tom\"', 'teacher', '\"Walt\"')),c =" + + " json_append('{\"school\":{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}'," + + " array('school.teacher', '[\"Tom\", \"Walt\"]'))| fields a, b, c | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "struct"), schema("b", "struct"), schema("c", "struct")); + + verifyDataRows( + actual, + rows( + gson.fromJson( + "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tomy\",\"rank\":5}]}", + Map.class), + gson.fromJson( + "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}", + Map.class), + gson.fromJson( + "{\"school\":{\"teacher\":[\"Alice\",[\"Tom\",\"Walt\"]],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}", + Map.class))); + } + + @Test + public void testJsonExtend() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =" + + " json_extend('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," + + " array('student', '{\"name\":\"Tommy\",\"rank\":5}')), b =" + + " json_extend('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," + + " array('teacher', '\"Tom\"', 'teacher', '\"Walt\"')),c =" + + " json_extend('{\"school\":{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}'," + + " array('school.teacher', '[\"Tom\", \"Walt\"]'))| fields a, b, c | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "struct"), schema("b", "struct"), schema("c", "struct")); + + verifyDataRows( + actual, + rows( + gson.fromJson( + "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tommy\",\"rank\":5}]}", + Map.class), + gson.fromJson( + "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}", + Map.class), + gson.fromJson( + "{\"school\":{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}", + Map.class))); + } } From df9ecbb64cb5c4482b6a82dd8485105fd32a4aab Mon Sep 17 00:00:00 2001 From: xinyual Date: Tue, 22 Apr 2025 11:33:18 +0800 Subject: [PATCH 06/50] fix IT Signed-off-by: xinyual --- .../function/jsonUDF/JsonDeleteFunctionImpl.java | 4 ---- .../standalone/CalcitePPLJsonBuiltinFunctionIT.java | 10 +++++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java index 24906cc4664..9392563ac8a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java @@ -55,10 +55,6 @@ public static Object eval(Object... args) throws JsonProcessingException { String jsonStr = (String) args[0]; Map jsonMap = objectMapper.readValue(jsonStr, Map.class); List keys = (List) args[1]; - if (keys.size() % 2 != 0) { - throw new RuntimeException( - "Json append function needs corresponding path and values, but current get: " + keys); - } for (String key : keys) { String[] keyParts = key.split("\\."); removeNestedKey(jsonMap, keyParts, 0); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index ae931baebf0..b0aac428012 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -55,7 +55,11 @@ public void testJsonObject() { verifySchema(actual, schema("a", "struct"), schema("b", "struct")); - verifyDataRows(actual, rows("[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]", null)); + verifyDataRows( + actual, + rows( + gson.fromJson("{\"key\":123.45}", Map.class), + gson.fromJson("{\"outer\":{\"inner\":123.45}}", Map.class))); } @Test @@ -68,7 +72,7 @@ public void testJsonArray() { verifySchema(actual, schema("a", "array")); - verifyDataRows(actual, rows("[1.0, 2.0, 0, -1.0, 1.1, -0.11]")); + verifyDataRows(actual, rows(List.of(1.0, 2.0, 0, -1.0, 1.1, -0.11))); } @Ignore( @@ -101,7 +105,7 @@ public void testToJsonString() { verifySchema(actual, schema("a", "string")); - verifyDataRows(actual, rows("[1.0, 2.0, 0.0, -1.0, 1.1, -0.11]")); + verifyDataRows(actual, rows("[1.0,2.0,0.0,-1.0,1.1,-0.11]")); } @Test From 5093024025398a78826f9a464fca640a6ab77884 Mon Sep 17 00:00:00 2001 From: xinyual Date: Tue, 22 Apr 2025 11:34:57 +0800 Subject: [PATCH 07/50] fix error message Signed-off-by: xinyual --- .../sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java | 2 +- .../sql/expression/function/jsonUDF/JsonSetFunctionImpl.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java index 3b6c282c028..7c2cacbdac4 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -56,7 +56,7 @@ public static Object eval(Object... args) throws JsonProcessingException { List keys = (List) args[1]; if (keys.size() % 2 != 0) { throw new RuntimeException( - "Json append function needs corresponding path and values, but current get: " + keys); + "Json extend function needs corresponding path and values, but current get: " + keys); } String resultStr = updateNestedJson(jsonStr, keys, JsonUtils::extendObjectValue); Map result = gson.fromJson(resultStr, Map.class); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java index ea0fe2c6599..53b1fe80c4e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -55,7 +55,7 @@ public static Object eval(Object... args) { List keys = (List) args[1]; if (keys.size() % 2 != 0) { throw new RuntimeException( - "Json append function needs corresponding path and values, but current get: " + keys); + "Json set function needs corresponding path and values, but current get: " + keys); } String resultStr = updateNestedJson(jsonStr, keys, (obj, key, value) -> obj.put(key, value)); Map result = gson.fromJson(resultStr, Map.class); From 9c8247c13c31c144ab4b7023cdb6806a11a3605d Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 23 Apr 2025 13:38:28 +0800 Subject: [PATCH 08/50] add json Signed-off-by: xinyual --- core/build.gradle | 1 + .../sql/ast/AbstractNodeVisitor.java | 5 ++ .../sql/ast/expression/LambdaFunction.java | 47 +++++++++++++++ .../sql/calcite/CalciteRexNodeVisitor.java | 23 ++++++++ .../function/BuiltinFunctionName.java | 7 +++ .../function/PPLBuiltinOperators.java | 3 + .../expression/function/PPLFuncImpTable.java | 2 + .../collectionUDF/ForallFunctionImpl.java | 58 +++++++++++++++++++ .../function/collectionUDF/LambdaUtils.java | 40 +++++++++++++ integ-test/build.gradle | 4 +- .../CalcitePPLJsonBuiltinFunctionIT.java | 16 +++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 8 +++ ppl/src/main/antlr/OpenSearchPPLParser.g4 | 12 ++++ .../sql/ppl/parser/AstExpressionBuilder.java | 10 ++++ 14 files changed, 234 insertions(+), 2 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/expression/LambdaFunction.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/LambdaUtils.java diff --git a/core/build.gradle b/core/build.gradle index dca34fcd303..33fcb11a6b4 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -64,6 +64,7 @@ dependencies { api project(':common') implementation "com.github.seancfoley:ipaddress:5.4.2" implementation "com.jayway.jsonpath:json-path:2.9.0" + implementation "com.googlecode.aviator:aviator:5.4.3" annotationProcessor('org.immutables:value:2.8.8') compileOnly('org.immutables:value-annotations:2.8.8') diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 6f432ce7bc7..dbae757561c 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -21,6 +21,7 @@ import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; import org.opensearch.sql.ast.expression.Interval; +import org.opensearch.sql.ast.expression.LambdaFunction; import org.opensearch.sql.ast.expression.Let; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Map; @@ -226,6 +227,10 @@ public T visitLet(Let node, C context) { return visitChildren(node, context); } + public T visitLambdaFunction(LambdaFunction node, C context) { + return visitChildren(node, context); + } + public T visitSort(Sort node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/LambdaFunction.java b/core/src/main/java/org/opensearch/sql/ast/expression/LambdaFunction.java new file mode 100644 index 00000000000..e7eab427652 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/expression/LambdaFunction.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.expression; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.ast.AbstractNodeVisitor; + +/** + * Expression node of lambda function. Params include function name (@funcName) and function + * arguments (@funcArgs) + */ +@Getter +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class LambdaFunction extends UnresolvedExpression { + private final UnresolvedExpression function; + private final List funcArgs; + + @Override + public List getChild() { + List children = new ArrayList<>(); + children.add(function); + children.addAll(funcArgs); + return children; + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitLambdaFunction(this, context); + } + + @Override + public String toString() { + return String.format( + "(%s) -> %s", + funcArgs.stream().map(Object::toString).collect(Collectors.joining(", ")), + function.toString()); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 4de88015fcb..41a6f7309a0 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -11,6 +11,7 @@ import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.TransferUserDefinedFunction; import java.math.BigDecimal; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; @@ -18,6 +19,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.sql.SqlOperator; @@ -37,6 +39,7 @@ import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.In; import org.opensearch.sql.ast.expression.Interval; +import org.opensearch.sql.ast.expression.LambdaFunction; import org.opensearch.sql.ast.expression.Let; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Not; @@ -330,6 +333,26 @@ public RexNode visitLet(Let node, CalcitePlanContext context) { return context.relBuilder.alias(expr, node.getVar().getField().toString()); } + @Override + public RexNode visitLambdaFunction(LambdaFunction node, CalcitePlanContext context) { + /* + RelDataType intType = context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); + RexNode xRef = context.rexBuilder.makeInputRef(intType, 0); + RexNode body = context.rexBuilder.makeCall( + SqlStdOperatorTable.PLUS, + xRef, + context.rexBuilder.makeBigintLiteral(BigDecimal.ONE) + ); + + */ + RexNode body = node.getFunction().accept(this, context); + List names = node.getFuncArgs(); + List args = new ArrayList<>(); + args.add(new RexLambdaRef(0, names.get(0).toString(), context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY))); + RexNode lambdaNode = context.rexBuilder.makeLambdaCall(body, args); + return lambdaNode; + } + @Override public RexNode visitFunction(Function node, CalcitePlanContext context) { List arguments = diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 6bf99a0d898..72170cbae1f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -163,6 +163,13 @@ public enum BuiltinFunctionName { LIKE(FunctionName.of("like")), NOT_LIKE(FunctionName.of("not like")), + /** LAMBDA Functions **/ + ARRAY_FORALL(FunctionName.of("forall")), + ARRAY_EXISTS(FunctionName.of("exists")), + ARRAY_FILTER(FunctionName.of("filter")), + ARRAY_TRANSFORM(FunctionName.of("transform")), + ARRAY_AGGREGATE(FunctionName.of("reduce")), + /** Aggregation Function. */ AVG(FunctionName.of("avg")), SUM(FunctionName.of("sum")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 094aa3f9676..c6409932ff8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -16,6 +16,7 @@ import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable; import org.apache.calcite.util.BuiltInMethod; +import org.opensearch.sql.expression.function.collectionUDF.ForallFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonArrayFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl; @@ -50,6 +51,8 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND"); public static final SqlOperator JSON_EXTEND = new JsonExtendFunctionImpl().toUDF("JSON_EXTEND"); + public static final SqlOperator FORALL = new ForallFunctionImpl().toUDF("FORALL"); + /** * Invoking an implementor registered in {@link RexImpTable}, need to use reflection since they're * all private Use method directly in {@link BuiltInMethod} if possible, most operators' diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 0756347b927..19cf7676f3f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -231,6 +231,8 @@ void populate() { registerOperator(JSON_APPEND, PPLBuiltinOperators.JSON_APPEND); registerOperator(JSON_EXTEND, PPLBuiltinOperators.JSON_EXTEND); + registerOperator(ARRAY_FORALL, PPLBuiltinOperators.FORALL); + // Register implementation. // Note, make the implementation an individual class if too complex. register( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java new file mode 100644 index 00000000000..6d6316ebc0e --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java @@ -0,0 +1,58 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.collectionUDF; + +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; + +import java.util.List; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexImpTable; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.expression.function.ImplementorUDF; + +public class ForallFunctionImpl extends ImplementorUDF { + public ForallFunctionImpl() { + super(new ForallImplementor(), NullPolicy.ANY); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return VARCHAR_FORCE_NULLABLE; + } + + public static class ForallImplementor implements NotNullImplementor { + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + ScalarFunctionImpl function = + (ScalarFunctionImpl) + ScalarFunctionImpl.create( + Types.lookupMethod(ForallFunctionImpl.class, "eval", Object[].class)); + return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); + } + } + + public static Object eval(Object... args) { + org.apache.calcite.linq4j.function.Function1 demo = (org.apache.calcite.linq4j.function.Function1) args[1]; + demo.apply(5); + List target = (List) args[0]; + String lambdaInString = (String) args[1]; + try { + LambdaUtils.SimpleLambda simpleLambda = new LambdaUtils.SimpleLambda(lambdaInString); + List parameters = simpleLambda.parameters(); + + } catch (Exception e) { + throw new RuntimeException(e); + } + return null; + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/LambdaUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/LambdaUtils.java new file mode 100644 index 00000000000..38aedce6278 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/LambdaUtils.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.collectionUDF; + +import com.googlecode.aviator.AviatorEvaluator; +import com.googlecode.aviator.Expression; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class LambdaUtils { + public static class SimpleLambda { + private Expression expression; + private List inputVariables; + + public SimpleLambda(String lambda) { + String[] lambdaParts = lambda.split("->"); + if (lambdaParts.length != 2) { + throw new IllegalArgumentException("Invalid lambda expression format"); + } + + String parameter = lambdaParts[0].trim(); + expression = AviatorEvaluator.compile(lambdaParts[1].trim()); + inputVariables = + Arrays.stream(parameter.split(",")).map(String::trim).collect(Collectors.toList()); + } + + public List parameters() { + return inputVariables; + } + + public Object execute(Map variableMap) { + return expression.execute(variableMap); + } + } +} diff --git a/integ-test/build.gradle b/integ-test/build.gradle index f661c5a50df..6be82a71faa 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -436,8 +436,8 @@ integTest { dependsOn ':opensearch-sql-plugin:bundlePlugin' if(getOSFamilyType() != "windows") { - dependsOn startPrometheus - finalizedBy stopPrometheus + //dependsOn startPrometheus + //finalizedBy stopPrometheus } // enable calcite codegen in IT diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index b0aac428012..03826d085f2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -309,4 +309,20 @@ public void testJsonExtend() { "{\"school\":{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}", Map.class))); } + + @Test + public void testForAll() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval array = array(1, -1, 2), result = forall(array, x -> x + 1) | fields result | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "struct")); + + verifyDataRows( + actual, + rows( + gson.fromJson("{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}", Map.class))); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index f1355b27b1c..6acf3a34eea 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -185,6 +185,7 @@ RT_SQR_PRTHS: ']'; SINGLE_QUOTE: '\''; DOUBLE_QUOTE: '"'; BACKTICK: '`'; +ARROW: '->'; // Operators. Bit @@ -363,6 +364,13 @@ JSON_DELETE: 'JSON_DELETE'; JSON_APPEND: 'JSON_APPEND'; JSON_EXTEND: 'JSON_EXTEND'; +// LAMBDA FUNCTIONS +//EXISTS: 'EXISTS'; +FORALL: 'FORALL'; +FILTER: 'FILTER'; +TRANSFORM: 'TRANSFORM'; +REDUCE: 'REDUCE'; + // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; NULLIF: 'NULLIF'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 60253dd812d..30e1231d749 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -417,6 +417,8 @@ valueExpression | timestampFunction # timestampFunctionCall | LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr | LT_SQR_PRTHS subSearch RT_SQR_PRTHS # scalarSubqueryExpr + | ident ARROW expression # lambda + | LT_PRTHS ident (COMMA ident)+ RT_PRTHS ARROW expression # lambda ; primaryExpression @@ -531,6 +533,7 @@ evalFunctionName | jsonFunctionName | geoipFunctionName | collectionFunctionName + | lambdaFunctionName ; functionArgs @@ -667,6 +670,14 @@ jsonFunctionName | JSON_EXTEND ; +lambdaFunctionName + : FORALL + | EXISTS + | FILTER + | TRANSFORM + | REDUCE + ; + dateTimeFunctionName : ADDDATE | ADDTIME @@ -1003,6 +1014,7 @@ keywordsCanBeId | patternMethod // commands assist keywords | IN + | ARROW | BETWEEN | EXISTS | SOURCE diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index cdf81bb4ca4..48d23fab5a3 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -44,6 +44,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -130,6 +131,15 @@ public UnresolvedExpression visitLogicalXor(LogicalXorContext ctx) { return new Xor(visit(ctx.left), visit(ctx.right)); } + @Override + public UnresolvedExpression visitLambda(OpenSearchPPLParser.LambdaContext ctx) { + + List arguments = ctx.ident().stream().map(x -> this.visitIdentifiers(Collections.singletonList(x))).collect( + Collectors.toList()); + UnresolvedExpression function = visitExpression(ctx.expression()); + return new LambdaFunction(function, arguments); + } + /** Comparison expression. */ @Override public UnresolvedExpression visitCompareExpr(CompareExprContext ctx) { From 0d985e8272512ac65c9ca6d450cbb4b1cdc2e63a Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 23 Apr 2025 14:00:46 +0800 Subject: [PATCH 09/50] try field Signed-off-by: xinyual --- .../opensearch/sql/calcite/CalciteRexNodeVisitor.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 41a6f7309a0..b00c5497521 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -335,17 +335,18 @@ public RexNode visitLet(Let node, CalcitePlanContext context) { @Override public RexNode visitLambdaFunction(LambdaFunction node, CalcitePlanContext context) { - /* + RelDataType intType = context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); RexNode xRef = context.rexBuilder.makeInputRef(intType, 0); + RexNode simpleRef2 = context.relBuilder.field(0); RexNode body = context.rexBuilder.makeCall( SqlStdOperatorTable.PLUS, - xRef, + simpleRef2, context.rexBuilder.makeBigintLiteral(BigDecimal.ONE) ); - */ - RexNode body = node.getFunction().accept(this, context); + + //RexNode body = node.getFunction().accept(this, context); List names = node.getFuncArgs(); List args = new ArrayList<>(); args.add(new RexLambdaRef(0, names.get(0).toString(), context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY))); From 1b7d4edbfcbc2bb529e088a0e3c72ca9ff5d098b Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 23 Apr 2025 15:49:38 +0800 Subject: [PATCH 10/50] add json Signed-off-by: xinyual --- .../sql/calcite/CalcitePlanContext.java | 13 ++++- .../sql/calcite/CalciteRexNodeVisitor.java | 25 +++++----- .../collectionUDF/ForallFunctionImpl.java | 17 +++---- .../standalone/CalciteArrayFunctionIT.java | 48 +++++++++++++++++++ .../CalcitePPLJsonBuiltinFunctionIT.java | 16 ------- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 +- .../opensearch/sql/ppl/parser/AstBuilder.java | 2 +- 7 files changed, 83 insertions(+), 40 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index 5e6d81a9f30..45698af1919 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -8,12 +8,16 @@ import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; import java.sql.Connection; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; import java.util.Stack; import java.util.function.BiFunction; import lombok.Getter; import lombok.Setter; import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.RelBuilder; @@ -30,6 +34,8 @@ public class CalcitePlanContext { public final ExtendedRexBuilder rexBuilder; public final FunctionProperties functionProperties; public final QueryType queryType; + @Getter + public Map temparolInputmap; @Getter @Setter private boolean isResolvingJoinCondition = false; @Getter @Setter private boolean isResolvingExistsSubquery = false; @@ -42,9 +48,14 @@ private CalcitePlanContext(FrameworkConfig config, QueryType queryType) { this.relBuilder = CalciteToolsHelper.create(config, TYPE_FACTORY, connection); this.rexBuilder = new ExtendedRexBuilder(relBuilder.getRexBuilder()); this.functionProperties = new FunctionProperties(QueryType.PPL); + this.temparolInputmap = new HashMap<>(); } - public RexNode resolveJoinCondition( + public void putTemparolInputmap(String name, RexLambdaRef input) { + this.temparolInputmap.put(name, input); + } + + public RexNode resolveJoinCondition( UnresolvedExpression expr, BiFunction transformFunction) { isResolvingJoinCondition = true; diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index b00c5497521..41ba2ccf5ca 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -13,12 +13,14 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlIntervalQualifier; @@ -64,6 +66,7 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.PPLFuncImpTable; @@ -275,6 +278,10 @@ public RexNode visitQualifiedName(QualifiedName node, CalcitePlanContext context // 2. resolve QualifiedName in non-join condition String qualifiedName = node.toString(); List currentFields = context.relBuilder.peek().getRowType().getFieldNames(); + Map map = context.getTemparolInputmap(); + if (map.containsKey(qualifiedName)) { + return map.get(qualifiedName); + } if (currentFields.contains(qualifiedName)) { // 2.1 resolve QualifiedName from stack top return context.relBuilder.field(qualifiedName); @@ -335,21 +342,13 @@ public RexNode visitLet(Let node, CalcitePlanContext context) { @Override public RexNode visitLambdaFunction(LambdaFunction node, CalcitePlanContext context) { - - RelDataType intType = context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); - RexNode xRef = context.rexBuilder.makeInputRef(intType, 0); - RexNode simpleRef2 = context.relBuilder.field(0); - RexNode body = context.rexBuilder.makeCall( - SqlStdOperatorTable.PLUS, - simpleRef2, - context.rexBuilder.makeBigintLiteral(BigDecimal.ONE) - ); - - - //RexNode body = node.getFunction().accept(this, context); List names = node.getFuncArgs(); List args = new ArrayList<>(); - args.add(new RexLambdaRef(0, names.get(0).toString(), context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY))); + for (int i = 0; i < names.size(); i++) { + context.putTemparolInputmap(names.get(i).toString(), new RexLambdaRef(i, names.get(i).toString(), context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY))); + args.add(new RexLambdaRef(i, names.get(i).toString(), context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY))); + } + RexNode body = node.getFunction().accept(this, context); RexNode lambdaNode = context.rexBuilder.makeLambdaCall(body, args); return lambdaNode; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java index 6d6316ebc0e..a1f111bb411 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java @@ -16,6 +16,7 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -26,7 +27,7 @@ public ForallFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; + return ReturnTypes.BOOLEAN; } public static class ForallImplementor implements NotNullImplementor { @@ -42,17 +43,17 @@ public Expression implement( } public static Object eval(Object... args) { - org.apache.calcite.linq4j.function.Function1 demo = (org.apache.calcite.linq4j.function.Function1) args[1]; - demo.apply(5); + org.apache.calcite.linq4j.function.Predicate1 lambdaFunction = (org.apache.calcite.linq4j.function.Predicate1) args[1]; List target = (List) args[0]; - String lambdaInString = (String) args[1]; try { - LambdaUtils.SimpleLambda simpleLambda = new LambdaUtils.SimpleLambda(lambdaInString); - List parameters = simpleLambda.parameters(); - + for (Object candidate: target) { + if (!(Boolean) lambdaFunction.apply(candidate)) { + return false; + } + } } catch (Exception e) { throw new RuntimeException(e); } - return null; + return true; } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java new file mode 100644 index 00000000000..ef899c6be66 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.standalone; + +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Map; + +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.MatcherUtils.*; +import static org.opensearch.sql.util.MatcherUtils.rows; + +public class CalciteArrayFunctionIT extends CalcitePPLIntegTestCase { + @Override + public void init() throws IOException { + super.init(); + loadIndex(Index.STATE_COUNTRY); + loadIndex(Index.STATE_COUNTRY_WITH_NULL); + loadIndex(Index.DATE_FORMATS); + loadIndex(Index.BANK_WITH_NULL_VALUES); + loadIndex(Index.DATE); + loadIndex(Index.PEOPLE2); + loadIndex(Index.BANK); + } + + @Test + public void testForAll() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval array = array(1, -1, 2), result = forall(array, x -> x > 0) | fields result | head 1", + TEST_INDEX_BANK)); + + verifySchema(actual, schema("a", "struct")); + + verifyDataRows( + actual, + rows( + gson.fromJson("{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}", Map.class))); + } + +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 03826d085f2..b0aac428012 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -309,20 +309,4 @@ public void testJsonExtend() { "{\"school\":{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}", Map.class))); } - - @Test - public void testForAll() { - JSONObject actual = - executeQuery( - String.format( - "source=%s | eval array = array(1, -1, 2), result = forall(array, x -> x + 1) | fields result | head 1", - TEST_INDEX_PEOPLE2)); - - verifySchema(actual, schema("a", "struct")); - - verifyDataRows( - actual, - rows( - gson.fromJson("{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}", Map.class))); - } } diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 30e1231d749..fdefbbd667e 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -541,7 +541,7 @@ functionArgs ; functionArg - : (ident EQUAL)? expression + : (ident EQUAL)? valueExpression ; relevanceArg diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index e44b5752c05..e9a495b369c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -501,7 +501,7 @@ public UnresolvedPlan visitTableFunction(TableFunctionContext ctx) { arg -> { String argName = (arg.ident() != null) ? arg.ident().getText() : null; builder.add( - new UnresolvedArgument(argName, this.internalVisitExpression(arg.expression()))); + new UnresolvedArgument(argName, this.internalVisitExpression(arg.valueExpression()))); }); return new TableFunction(this.internalVisitExpression(ctx.qualifiedName()), builder.build()); } From 7f198720e90839e1c455f88693bbfa92e4c67b6d Mon Sep 17 00:00:00 2001 From: xinyual Date: Tue, 13 May 2025 14:54:12 +0800 Subject: [PATCH 11/50] update to splunk for extract Signed-off-by: xinyual --- .../jsonUDF/JsonExtractFunctionImpl.java | 2 + .../function/jsonUDF/JsonUtils.java | 41 ++++++++++++++++++ .../CalcitePPLJsonBuiltinFunctionIT.java | 43 +++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index db2ed0e7b45..e439f0b253a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -6,6 +6,7 @@ package org.opensearch.sql.expression.function.jsonUDF; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.convertToJsonPath; import com.jayway.jsonpath.JsonPath; import java.util.List; @@ -48,6 +49,7 @@ public static Object eval(Object... args) { } String value = (String) args[0]; String path = (String) args[1]; + path = convertToJsonPath(path); try { Object result = JsonPath.read(value, path); result = result != null ? result.toString() : null; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java index 84b6ad1c5f5..7084ac22977 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java @@ -161,4 +161,45 @@ static void removeNestedKey(Object currentObj, String[] keyParts, int depth) { } } } + + /** + * + * @param input candidate json path like a.b{}.c{2} + * @return the normalized json path like $.a.b[*].c[2] + */ + public static String convertToJsonPath(String input) { + if (input == null || input.isEmpty()) return "$"; + + StringBuilder sb = new StringBuilder("$"); + int i = 0; + while (i < input.length()) { + char c = input.charAt(i); + + if (c == '{') { + // 处理 {...} 为数组访问或通配符 + int end = input.indexOf('}', i); + if (end == -1) throw new IllegalArgumentException("Unmatched { in input"); + + String index = input.substring(i + 1, end).trim(); + if (index.isEmpty()) { + sb.append("[*]"); + } else { + sb.append("[").append(index).append("]"); + } + i = end + 1; + } else if (c == '.') { + sb.append("."); + i++; + } else { + // 读取字段名 + int start = i; + while (i < input.length() && input.charAt(i) != '.' && input.charAt(i) != '{') { + i++; + } + sb.append(input, start, i); + } + } + + return sb.toString(); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index b0aac428012..2b500f8771f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -123,6 +123,7 @@ public void testJsonArrayLength() { verifyDataRows(actual, rows(4, 5, null)); } + @Ignore @Test public void testJsonExtract() { JSONObject actual = @@ -144,6 +145,48 @@ public void testJsonExtract() { verifyDataRows(actual, rows("b", "2", "[1,2]", null)); } + @Test + public void testJsonExtractWithNewPath() { + String candidate = "[\n" + + " {\n" + + " \"name\": \"London\",\n" + + " \"Bridges\": [\n" + + " { \"name\": \"Tower Bridge\", \"length\": 801 },\n" + + " { \"name\": \"Millennium Bridge\", \"length\": 1066 }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"name\": \"Venice\",\n" + + " \"Bridges\": [\n" + + " { \"name\": \"Rialto Bridge\", \"length\": 157 },\n" + + " { \"name\": \"Bridge of Sighs\", \"length\": 36 },\n" + + " { \"name\": \"Ponte della Paglia\" }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"name\": \"San Francisco\",\n" + + " \"Bridges\": [\n" + + " { \"name\": \"Golden Gate Bridge\", \"length\": 8981 },\n" + + " { \"name\": \"Bay Bridge\", \"length\": 23556 }\n" + + " ]\n" + + " }\n" + + " ]"; + JSONObject actual = + executeQuery( + String.format( + "source=%s | head 1 | eval a = json_extract('%s', '{}'), b= json_extract('%s', '{2}.Bridges{0}.length'), c= json_extract('%s', '{2}'), d=json_extract('%s', '{2}.Bridges{0}')| fields a, b, c, d | head 1", + TEST_INDEX_PEOPLE2, candidate, candidate, candidate, candidate)); + + verifySchema( + actual, + schema("a", "string"), + schema("b", "string"), + schema("c", "string"), + schema("d", "string")); + + verifyDataRows(actual, rows("b", "2", "[1,2]", null)); + } + @Test public void testJsonKeys() { JSONObject actual = From 84e74b976732c654bb3ab46c830a0b8c952eda76 Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 14 May 2025 11:11:35 +0800 Subject: [PATCH 12/50] align with splunk Signed-off-by: xinyual --- .../sql/calcite/CalcitePlanContext.java | 6 +- .../sql/calcite/CalciteRexNodeVisitor.java | 15 ++- .../function/BuiltinFunctionName.java | 2 +- .../collectionUDF/ForallFunctionImpl.java | 7 +- .../jsonUDF/JsonAppendFunctionImpl.java | 70 ++++++++++- .../jsonUDF/JsonDeleteFunctionImpl.java | 27 ++-- .../jsonUDF/JsonExtendFunctionImpl.java | 7 +- .../jsonUDF/JsonExtractFunctionImpl.java | 3 +- .../function/jsonUDF/JsonSetFunctionImpl.java | 69 ++++++++++- .../function/jsonUDF/JsonUtils.java | 96 ++++++++++++++- .../standalone/CalciteArrayFunctionIT.java | 67 +++++----- .../CalcitePPLJsonBuiltinFunctionIT.java | 116 ++++++++++-------- .../opensearch/sql/ppl/parser/AstBuilder.java | 3 +- .../sql/ppl/parser/AstExpressionBuilder.java | 6 +- 14 files changed, 378 insertions(+), 116 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index 45698af1919..ed418ad8507 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -16,7 +16,6 @@ import lombok.Getter; import lombok.Setter; import org.apache.calcite.rex.RexCorrelVariable; -import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.FrameworkConfig; @@ -34,8 +33,7 @@ public class CalcitePlanContext { public final ExtendedRexBuilder rexBuilder; public final FunctionProperties functionProperties; public final QueryType queryType; - @Getter - public Map temparolInputmap; + @Getter public Map temparolInputmap; @Getter @Setter private boolean isResolvingJoinCondition = false; @Getter @Setter private boolean isResolvingExistsSubquery = false; @@ -55,7 +53,7 @@ public void putTemparolInputmap(String name, RexLambdaRef input) { this.temparolInputmap.put(name, input); } - public RexNode resolveJoinCondition( + public RexNode resolveJoinCondition( UnresolvedExpression expr, BiFunction transformFunction) { isResolvingJoinCondition = true; diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 41ba2ccf5ca..b68612a196e 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -20,7 +20,6 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlIntervalQualifier; @@ -66,7 +65,6 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.SemanticCheckException; -import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.PPLFuncImpTable; @@ -345,8 +343,17 @@ public RexNode visitLambdaFunction(LambdaFunction node, CalcitePlanContext conte List names = node.getFuncArgs(); List args = new ArrayList<>(); for (int i = 0; i < names.size(); i++) { - context.putTemparolInputmap(names.get(i).toString(), new RexLambdaRef(i, names.get(i).toString(), context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY))); - args.add(new RexLambdaRef(i, names.get(i).toString(), context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY))); + context.putTemparolInputmap( + names.get(i).toString(), + new RexLambdaRef( + i, + names.get(i).toString(), + context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY))); + args.add( + new RexLambdaRef( + i, + names.get(i).toString(), + context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY))); } RexNode body = node.getFunction().accept(this, context); RexNode lambdaNode = context.rexBuilder.makeLambdaCall(body, args); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 72170cbae1f..1ddda5b199f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -163,7 +163,7 @@ public enum BuiltinFunctionName { LIKE(FunctionName.of("like")), NOT_LIKE(FunctionName.of("not like")), - /** LAMBDA Functions **/ + /** LAMBDA Functions * */ ARRAY_FORALL(FunctionName.of("forall")), ARRAY_EXISTS(FunctionName.of("exists")), ARRAY_FILTER(FunctionName.of("filter")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java index a1f111bb411..e2da4acee61 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java @@ -5,8 +5,6 @@ package org.opensearch.sql.expression.function.collectionUDF; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; - import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; @@ -43,10 +41,11 @@ public Expression implement( } public static Object eval(Object... args) { - org.apache.calcite.linq4j.function.Predicate1 lambdaFunction = (org.apache.calcite.linq4j.function.Predicate1) args[1]; + org.apache.calcite.linq4j.function.Predicate1 lambdaFunction = + (org.apache.calcite.linq4j.function.Predicate1) args[1]; List target = (List) args[0]; try { - for (Object candidate: target) { + for (Object candidate : target) { if (!(Boolean) lambdaFunction.apply(candidate)) { return false; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java index 6ee7c64d783..a159aac959b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -9,6 +9,15 @@ import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.PathNotFoundException; +import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import java.util.List; import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -53,13 +62,70 @@ public Expression implement( public static Object eval(Object... args) throws JsonProcessingException { String jsonStr = (String) args[0]; - List keys = (List) args[1]; + List keys = collectKeyValuePair(args); if (keys.size() % 2 != 0) { throw new RuntimeException( "Json append function needs corresponding path and values, but current get: " + keys); } - String resultStr = updateNestedJson(jsonStr, keys, JsonUtils::appendObjectValue); + String resultStr = jsonAppendIfArray(jsonStr, keys, false); Map result = gson.fromJson(resultStr, Map.class); return result; } + + public static String jsonAppendIfArray(String json, List pathValueMap, boolean isExtend) { + ObjectMapper mapper = new ObjectMapper(); + try { + JsonNode tree = mapper.readTree(json); + + Configuration conf = + Configuration.builder() + .jsonProvider(new JacksonJsonNodeJsonProvider()) + .mappingProvider(new JacksonMappingProvider()) + .build(); + + DocumentContext context = JsonPath.using(conf).parse(tree); + + for (int index = 0; index < pathValueMap.size(); index += 2) { + String jsonPath = pathValueMap.get(index).toString(); + Object valueToAppend = pathValueMap.get(index + 1); + JsonNode targets; + try { + targets = context.read(jsonPath); + } catch (PathNotFoundException e) { + continue; + } + if (JsonPath.isPathDefinite(jsonPath)) { + if (targets instanceof ArrayNode arrayNode) { + if (isExtend && valueToAppend instanceof List list) { + for (Object value : list) { + arrayNode.addPOJO(value); + } + } else { + arrayNode.addPOJO(valueToAppend); + } + } + } else { + // Some * inside. an arrayNode returned + for (int i = 0; i < targets.size(); i++) { + JsonNode target = targets.get(i); + if (target instanceof ArrayNode arrayNode) { + if (isExtend && valueToAppend instanceof List list) { + for (Object value : list) { + arrayNode.addPOJO(value); + } + } else { + arrayNode.addPOJO(valueToAppend); + } + } + } + } + } + return mapper.writeValueAsString(tree); + } catch (Exception e) { + if (e instanceof PathNotFoundException) { + return json; + } + throw new RuntimeException("Failed to process JSON", e); + } + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java index 9392563ac8a..05d98c9b0a0 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java @@ -9,6 +9,11 @@ import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.PathNotFoundException; import java.util.List; import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -53,13 +58,21 @@ public Expression implement( public static Object eval(Object... args) throws JsonProcessingException { String jsonStr = (String) args[0]; - Map jsonMap = objectMapper.readValue(jsonStr, Map.class); - List keys = (List) args[1]; - for (String key : keys) { - String[] keyParts = key.split("\\."); - removeNestedKey(jsonMap, keyParts, 0); + List jsonPaths = collectKeyValuePair(args); + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = mapper.readTree(jsonStr); + DocumentContext ctx = JsonPath.parse(jsonStr); + for (Object originalPath : jsonPaths) { + String jsonPath = convertToJsonPath(originalPath.toString()); + try { + Object matches = ctx.read(jsonPath); // verify whether it's a valid path + } catch (PathNotFoundException e) { + return gson.fromJson(jsonStr, Map.class); + } + // Resolve path tokens + PathTokenizer tokenizer = new PathTokenizer(jsonPath); + root = deletePath(root, tokenizer); } - Map result = gson.fromJson(objectMapper.writeValueAsString(jsonMap), Map.class); - return result; + return gson.fromJson(mapper.writeValueAsString(root), Map.class); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java index 7c2cacbdac4..e456942f572 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -6,7 +6,8 @@ package org.opensearch.sql.expression.function.jsonUDF; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.updateNestedJson; +import static org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl.jsonAppendIfArray; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.collectKeyValuePair; import com.fasterxml.jackson.core.JsonProcessingException; import java.util.List; @@ -53,12 +54,12 @@ public Expression implement( public static Object eval(Object... args) throws JsonProcessingException { String jsonStr = (String) args[0]; - List keys = (List) args[1]; + List keys = collectKeyValuePair(args); if (keys.size() % 2 != 0) { throw new RuntimeException( "Json extend function needs corresponding path and values, but current get: " + keys); } - String resultStr = updateNestedJson(jsonStr, keys, JsonUtils::extendObjectValue); + String resultStr = jsonAppendIfArray(jsonStr, keys, true); Map result = gson.fromJson(resultStr, Map.class); return result; } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index e439f0b253a..c7b30db1827 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -6,6 +6,7 @@ package org.opensearch.sql.expression.function.jsonUDF; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.convertToJsonPath; import com.jayway.jsonpath.JsonPath; @@ -52,7 +53,7 @@ public static Object eval(Object... args) { path = convertToJsonPath(path); try { Object result = JsonPath.read(value, path); - result = result != null ? result.toString() : null; + result = result != null ? gson.toJson(result) : null; return result; } catch (Exception e) { return null; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java index 53b1fe80c4e..45328a72197 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -6,8 +6,17 @@ package org.opensearch.sql.expression.function.jsonUDF; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.updateNestedJson; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.PathNotFoundException; +import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import java.util.List; import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -52,13 +61,67 @@ public Expression implement( public static Object eval(Object... args) { String jsonStr = (String) args[0]; - List keys = (List) args[1]; + List keys = collectKeyValuePair(args); if (keys.size() % 2 != 0) { throw new RuntimeException( "Json set function needs corresponding path and values, but current get: " + keys); } - String resultStr = updateNestedJson(jsonStr, keys, (obj, key, value) -> obj.put(key, value)); + String resultStr = jsonSetIfParentObject(jsonStr, keys); Map result = gson.fromJson(resultStr, Map.class); return result; } + + public static String jsonSetIfParentObject(String json, List fullPathToValue) { + ObjectMapper mapper = new ObjectMapper(); + try { + JsonNode root = mapper.readTree(json); + + Configuration conf = + Configuration.builder() + .jsonProvider(new JacksonJsonNodeJsonProvider()) + .mappingProvider(new JacksonMappingProvider()) + .build(); + + DocumentContext context = JsonPath.using(conf).parse(root); + + for (int index = 0; index < fullPathToValue.size(); index += 2) { + String fullPath = convertToJsonPath(fullPathToValue.get(index).toString()); + Object value = fullPathToValue.get(index + 1); + + // Split: "$.a.b.d" -> parentPath = "$.a.b", field = "d" + int lastDot = fullPath.lastIndexOf('.'); + if (lastDot <= 1 || lastDot == fullPath.length() - 1) { + continue; // Invalid path + } + + String parentPath = fullPath.substring(0, lastDot); + String fieldName = fullPath.substring(lastDot + 1); + + JsonNode targets; + try { + targets = context.read(parentPath); + } catch (PathNotFoundException e) { + continue; // parent path not found + } + + if (JsonPath.isPathDefinite(parentPath)) { + if (targets instanceof ObjectNode objectNode) { + objectNode.set(fieldName, mapper.valueToTree(value)); + } + } else { + // Some * inside. an arrayNode returned + for (int i = 0; i < targets.size(); i++) { + JsonNode target = targets.get(i); + if (target instanceof ObjectNode objectNode) { + objectNode.set(fieldName, mapper.valueToTree(value)); + } + } + } + } + + return mapper.writeValueAsString(root); + } catch (Exception e) { + throw new RuntimeException("jsonSetIfParentObject failed", e); + } + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java index 7084ac22977..32be405e1b9 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java @@ -5,7 +5,10 @@ package org.opensearch.sql.expression.function.jsonUDF; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -163,14 +166,13 @@ static void removeNestedKey(Object currentObj, String[] keyParts, int depth) { } /** - * * @param input candidate json path like a.b{}.c{2} * @return the normalized json path like $.a.b[*].c[2] */ public static String convertToJsonPath(String input) { if (input == null || input.isEmpty()) return "$"; - StringBuilder sb = new StringBuilder("$"); + StringBuilder sb = new StringBuilder("$."); int i = 0; while (i < input.length()) { char c = input.charAt(i); @@ -202,4 +204,94 @@ public static String convertToJsonPath(String input) { return sb.toString(); } + + static JsonNode deletePath(JsonNode node, PathTokenizer tokenizer) { + if (!tokenizer.hasNext()) return node; + + PathToken token = tokenizer.next(); + + if (token.key.equals("$")) { + return deletePath(node, tokenizer); + } + if (node instanceof ArrayNode arr && token.isIndex) { + if (token.key.equals("*")) { + for (int i = arr.size() - 1; i >= 0; i--) { + deletePath(arr.get(i), tokenizer.cloneFromNext()); + } + } else { + int idx = Integer.parseInt(token.key); + if (!tokenizer.hasNext()) { + if (idx >= 0 && idx < arr.size()) arr.remove(idx); + } else { + if (idx >= 0 && idx < arr.size()) { + deletePath(arr.get(idx), tokenizer); + } + } + } + } else if (node instanceof ObjectNode obj && !token.isIndex) { + if (!tokenizer.hasNext()) { + obj.remove(token.key); + } else if (obj.has(token.key)) { + deletePath(obj.get(token.key), tokenizer); + } + } + + return node; + } + + // Tokenizer for JSONPath like $[0].cities[1] + public static class PathTokenizer { + private final List tokens; + private int index = 0; + + public PathTokenizer(String path) { + this.tokens = new ArrayList<>(); + + // normalize brackets (a[1] => a.[1]) + String normalized = path.replaceAll("\\[", ".["); + + for (String raw : normalized.split("\\.")) { + if (raw.isEmpty()) continue; + + if (raw.startsWith("[") && raw.endsWith("]")) { + tokens.add(new PathToken(raw.substring(1, raw.length() - 1), true)); + } else { + tokens.add(new PathToken(raw, false)); + } + } + } + + public boolean hasNext() { + return index < tokens.size(); + } + + public PathToken next() { + return tokens.get(index++); + } + + public PathTokenizer cloneFromNext() { + PathTokenizer clone = new PathTokenizer(""); + clone.tokens.addAll(this.tokens); + clone.index = this.index; + return clone; + } + } + + static class PathToken { + public final String key; + public final boolean isIndex; + + public PathToken(String key, boolean isIndex) { + this.key = key; + this.isIndex = isIndex; + } + } + + public static List collectKeyValuePair(Object... args) { + List result = new ArrayList<>(); + for (int i = 1; i < args.length; i++) { + result.add(args[i]); + } + return result; + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java index ef899c6be66..6b08f1e7695 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java @@ -5,44 +5,43 @@ package org.opensearch.sql.calcite.standalone; -import org.json.JSONObject; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.Map; - import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.util.MatcherUtils.*; import static org.opensearch.sql.util.MatcherUtils.rows; -public class CalciteArrayFunctionIT extends CalcitePPLIntegTestCase { - @Override - public void init() throws IOException { - super.init(); - loadIndex(Index.STATE_COUNTRY); - loadIndex(Index.STATE_COUNTRY_WITH_NULL); - loadIndex(Index.DATE_FORMATS); - loadIndex(Index.BANK_WITH_NULL_VALUES); - loadIndex(Index.DATE); - loadIndex(Index.PEOPLE2); - loadIndex(Index.BANK); - } - - @Test - public void testForAll() { - JSONObject actual = - executeQuery( - String.format( - "source=%s | eval array = array(1, -1, 2), result = forall(array, x -> x > 0) | fields result | head 1", - TEST_INDEX_BANK)); - - verifySchema(actual, schema("a", "struct")); - - verifyDataRows( - actual, - rows( - gson.fromJson("{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}", Map.class))); - } +import java.io.IOException; +import java.util.Map; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +public class CalciteArrayFunctionIT extends CalcitePPLIntegTestCase { + @Override + public void init() throws IOException { + super.init(); + loadIndex(Index.STATE_COUNTRY); + loadIndex(Index.STATE_COUNTRY_WITH_NULL); + loadIndex(Index.DATE_FORMATS); + loadIndex(Index.BANK_WITH_NULL_VALUES); + loadIndex(Index.DATE); + loadIndex(Index.PEOPLE2); + loadIndex(Index.BANK); + } + + @Test + public void testForAll() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval array = array(1, -1, 2), result = forall(array, x -> x > 0) |" + + " fields result | head 1", + TEST_INDEX_BANK)); + + verifySchema(actual, schema("a", "struct")); + + verifyDataRows( + actual, + rows( + gson.fromJson("{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}", Map.class))); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 2b500f8771f..605cfb6c290 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -147,44 +147,47 @@ public void testJsonExtract() { @Test public void testJsonExtractWithNewPath() { - String candidate = "[\n" + - " {\n" + - " \"name\": \"London\",\n" + - " \"Bridges\": [\n" + - " { \"name\": \"Tower Bridge\", \"length\": 801 },\n" + - " { \"name\": \"Millennium Bridge\", \"length\": 1066 }\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\": \"Venice\",\n" + - " \"Bridges\": [\n" + - " { \"name\": \"Rialto Bridge\", \"length\": 157 },\n" + - " { \"name\": \"Bridge of Sighs\", \"length\": 36 },\n" + - " { \"name\": \"Ponte della Paglia\" }\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"name\": \"San Francisco\",\n" + - " \"Bridges\": [\n" + - " { \"name\": \"Golden Gate Bridge\", \"length\": 8981 },\n" + - " { \"name\": \"Bay Bridge\", \"length\": 23556 }\n" + - " ]\n" + - " }\n" + - " ]"; + String candidate = + "[\n" + + "{\n" + + "\"name\":\"London\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Tower Bridge\",\"length\":801.0},\n" + + "{\"name\":\"Millennium Bridge\",\"length\":1066.0}\n" + + "]\n" + + "},\n" + + "{\n" + + "\"name\":\"Venice\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Rialto Bridge\",\"length\":157.0},\n" + + "{\"name\":\"Bridge of Sighs\",\"length\":36.0},\n" + + "{\"name\":\"Ponte della Paglia\"}\n" + + "]\n" + + "},\n" + + "{\n" + + "\"name\": \"San Francisco\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0},\n" + + "{\"name\":\"Bay Bridge\", \"length\":23556.0}\n" + + "]\n" + + "}\n" + + "]"; JSONObject actual = - executeQuery( - String.format( - "source=%s | head 1 | eval a = json_extract('%s', '{}'), b= json_extract('%s', '{2}.Bridges{0}.length'), c= json_extract('%s', '{2}'), d=json_extract('%s', '{2}.Bridges{0}')| fields a, b, c, d | head 1", - TEST_INDEX_PEOPLE2, candidate, candidate, candidate, candidate)); + executeQuery( + String.format( + "source=%s | head 1 | eval a = json_extract('%s', '{}'), b= json_extract('%s'," + + " '{2}.Bridges{0}.length'), d=json_extract('%s', '{2}.Bridges{0}')| fields a," + + " b, d | head 1", + TEST_INDEX_PEOPLE2, candidate, candidate, candidate, candidate)); - verifySchema( - actual, - schema("a", "string"), - schema("b", "string"), - schema("c", "string"), - schema("d", "string")); + verifySchema(actual, schema("a", "string"), schema("b", "string"), schema("d", "string")); - verifyDataRows(actual, rows("b", "2", "[1,2]", null)); + verifyDataRows( + actual, + rows( + gson.toJson(gson.fromJson(candidate, List.class)), + "8981.0", + "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0}")); } @Test @@ -234,13 +237,13 @@ public void testJsonSet() { JSONObject actual = executeQuery( String.format( - "source=%s | eval a =json_set('{\"a\":[{\"b\":1},{\"b\":2}]}', array('a.b', '3'))|" + "source=%s | eval a =json_set('{\"a\":[{\"b\":1},{\"b\":2}]}', 'a{}.b', '3')|" + " fields a | head 1", TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "struct")); - verifyDataRows(actual, rows(gson.fromJson("{\"a\":[{\"b\":3},{\"b\":3}]}", Map.class))); + verifyDataRows(actual, rows(gson.fromJson("{\"a\":[{\"b\":\"3\"},{\"b\":\"3\"}]}", Map.class))); } @Test @@ -250,7 +253,7 @@ public void testJsonDelete() { String.format( "source=%s | eval a" + " =json_delete('{\"account_number\":1,\"balance\":39225,\"age\":32,\"gender\":\"M\"}'," - + " array('age','gender'))| fields a | head 1", + + " 'age','gender')| fields a | head 1", TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "struct")); @@ -266,7 +269,7 @@ public void testJsonDeleteWithNested() { String.format( "source=%s | eval a" + " =json_delete('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}'," - + " array('f2.f3')) | fields a | head 1", + + " 'f2.f3') | fields a | head 1", TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "struct")); @@ -276,13 +279,30 @@ public void testJsonDeleteWithNested() { } @Test - public void testJsonDeleteWithNestedAndArry() { + public void testJsonDeleteWithNestedNothing() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a" + + " =json_delete('{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}'," + + " 'f2.f100') | fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "struct")); + + verifyDataRows( + actual, + rows(gson.fromJson("{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}", Map.class))); + } + + @Test + public void testJsonDeleteWithNestedAndArray() { JSONObject actual = executeQuery( String.format( "source=%s | eval a" - + " =json_delete('{\"teacher\":\"Alice\",\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}',array('teacher'," - + " 'student.rank')) | fields a | head 1", + + " =json_delete('{\"teacher\":\"Alice\",\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}','teacher'," + + " 'student{}.rank') | fields a | head 1", TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "struct")); @@ -300,11 +320,11 @@ public void testJsonAppend() { String.format( "source=%s | eval a" + " =json_append('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," - + " array('student', '{\"name\":\"Tomy\",\"rank\":5}')), b =" + + " 'student', json_object(\"name\", \"Tomy\",\"rank\", 5)), b =" + " json_append('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," - + " array('teacher', '\"Tom\"', 'teacher', '\"Walt\"')),c =" + + " 'teacher', 'Tom', 'teacher', 'Walt'),c =" + " json_append('{\"school\":{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}'," - + " array('school.teacher', '[\"Tom\", \"Walt\"]'))| fields a, b, c | head 1", + + " 'school.teacher', json_array(\"Tom\", \"Walt\"))| fields a, b, c | head 1", TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "struct"), schema("b", "struct"), schema("c", "struct")); @@ -330,11 +350,11 @@ public void testJsonExtend() { String.format( "source=%s | eval a =" + " json_extend('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," - + " array('student', '{\"name\":\"Tommy\",\"rank\":5}')), b =" + + " 'student', json_object(\"name\", \"Tommy\",\"rank\", 5)), b =" + " json_extend('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," - + " array('teacher', '\"Tom\"', 'teacher', '\"Walt\"')),c =" + + " 'teacher', 'Tom', 'teacher', 'Walt'),c =" + " json_extend('{\"school\":{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}'," - + " array('school.teacher', '[\"Tom\", \"Walt\"]'))| fields a, b, c | head 1", + + " 'school.teacher', array(\"Tom\", \"Walt\"))| fields a, b, c | head 1", TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "struct"), schema("b", "struct"), schema("c", "struct")); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index e9a495b369c..928f767cd0a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -501,7 +501,8 @@ public UnresolvedPlan visitTableFunction(TableFunctionContext ctx) { arg -> { String argName = (arg.ident() != null) ? arg.ident().getText() : null; builder.add( - new UnresolvedArgument(argName, this.internalVisitExpression(arg.valueExpression()))); + new UnresolvedArgument( + argName, this.internalVisitExpression(arg.valueExpression()))); }); return new TableFunction(this.internalVisitExpression(ctx.qualifiedName()), builder.build()); } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 48d23fab5a3..d22115deef8 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -134,8 +134,10 @@ public UnresolvedExpression visitLogicalXor(LogicalXorContext ctx) { @Override public UnresolvedExpression visitLambda(OpenSearchPPLParser.LambdaContext ctx) { - List arguments = ctx.ident().stream().map(x -> this.visitIdentifiers(Collections.singletonList(x))).collect( - Collectors.toList()); + List arguments = + ctx.ident().stream() + .map(x -> this.visitIdentifiers(Collections.singletonList(x))) + .collect(Collectors.toList()); UnresolvedExpression function = visitExpression(ctx.expression()); return new LambdaFunction(function, arguments); } From 89df43177bf35d889d28c8229fdab95b791991c6 Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 14 May 2025 13:09:01 +0800 Subject: [PATCH 13/50] fi original IT Signed-off-by: xinyual --- .../CalcitePPLJsonBuiltinFunctionIT.java | 12 +++++++ .../opensearch/sql/ppl/JsonFunctionsIT.java | 34 +++++++++++++++---- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 605cfb6c290..04b72e635b3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -28,6 +28,7 @@ public void init() throws IOException { loadIndex(Index.DATE); loadIndex(Index.PEOPLE2); loadIndex(Index.BANK); + loadIndex(Index.JSON_TEST); } @Test @@ -372,4 +373,15 @@ public void testJsonExtend() { "{\"school\":{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}", Map.class))); } + + @Test + public void test_cast_json() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" + + " | fields test_name", + TEST_INDEX_JSON_TEST)); + assertEquals(1, 1); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 9bd40e66535..08dc1651a47 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -14,6 +14,8 @@ import java.io.IOException; import java.util.List; import java.util.Map; + +import com.google.gson.Gson; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -71,7 +73,13 @@ public void test_cast_json() throws IOException { "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" + " | fields test_name, casted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + String jsonType; + if (isCalciteEnabled()) { + jsonType = "string"; + } else { + jsonType = "undefined"; + } + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, jsonType)); verifyDataRows( result, rows( @@ -102,19 +110,33 @@ public void test_json() throws IOException { "source=%s | where json_valid(json_string) | eval casted=json(json_string) | fields" + " test_name, casted", TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "undefined")); + String jsonType; + if (isCalciteEnabled()) { + jsonType = "string"; + } else { + jsonType = "undefined"; + } + verifySchema(result, schema("test_name", null, "string"), schema("casted", null, jsonType)); JSONObject firstRow = new JSONObject(Map.of("c", 2)); + Object nestedArray = new JSONArray(List.of(1, 2, 3, Map.of("true", true, "number", 123))); + if (isCalciteEnabled()) { + nestedArray = nestedArray.toString(); + } + Object nestedObject = new JSONObject( + Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(Boolean.FALSE, 3))); + if (isCalciteEnabled()) { + nestedObject = new Gson().toJson(Map.of("d", List.of(Boolean.FALSE, 3), "a", "1", "b", Map.of("c", "3"))); + } verifyDataRows( result, rows( - "json nested object", - new JSONObject( - Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(Boolean.FALSE, 3)))), + "json nested object", nestedObject + ), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), rows( "json nested array", - new JSONArray(List.of(1, 2, 3, Map.of("true", true, "number", 123)))), + nestedArray), rows("json scalar string", "abc"), rows("json scalar int", 1234), rows("json scalar float", 12.34), From 1fd7fea9cf1cdca636c8103ca818e1d0645b17a6 Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 14 May 2025 16:56:27 +0800 Subject: [PATCH 14/50] change array and json function Signed-off-by: xinyual --- .../jsonUDF/JsonArrayFunctionImpl.java | 27 ++++++++++-------- .../function/jsonUDF/JsonFunctionImpl.java | 19 +++++++------ .../CalcitePPLJsonBuiltinFunctionIT.java | 28 ++++++++----------- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java index 08f876db71d..61bfab676d5 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java @@ -43,14 +43,14 @@ public JsonArrayFunctionImpl() { public SqlReturnTypeInference getReturnTypeInference() { return sqlOperatorBinding -> { RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - List argTypes = sqlOperatorBinding.collectOperandTypes(); - RelDataType commonType = typeFactory.leastRestrictive(argTypes); - if (commonType == null) { - throw new IllegalArgumentException( - "All arguments in json array cannot be converted into one common types"); - } + //List argTypes = sqlOperatorBinding.collectOperandTypes(); + //RelDataType commonType = typeFactory.leastRestrictive(argTypes); + //if (commonType == null) { + // throw new IllegalArgumentException( + // "All arguments in json array cannot be converted into one common types"); + //} return createArrayType( - typeFactory, typeFactory.createTypeWithNullability(commonType, true), true); + typeFactory, typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.ANY), true), true); }; } @@ -58,16 +58,18 @@ public static class JsonArrayImplementor implements NotNullImplementor { @Override public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { - RelDataType realType = call.getType().getComponentType(); - List newArgs = new ArrayList<>(translatedOperands); - assert realType != null; - newArgs.add(Expressions.constant(realType.getSqlTypeName())); + //RelDataType realType = call.getType().getComponentType(); + //List newArgs = new ArrayList<>(translatedOperands); + //assert realType != null; + //newArgs.add(Expressions.constant(realType.getSqlTypeName())); return Expressions.call( - Types.lookupMethod(JsonArrayFunctionImpl.class, "eval", Object[].class), newArgs); + Types.lookupMethod(JsonArrayFunctionImpl.class, "eval", Object[].class), translatedOperands); } } public static Object eval(Object... args) { + return Arrays.asList(args); + /* SqlTypeName targetType = (SqlTypeName) args[args.length - 1]; switch (targetType) { case DOUBLE: @@ -86,5 +88,6 @@ public static Object eval(Object... args) { default: return Arrays.asList(args).subList(0, args.length - 1); } + */ } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java index 43f9bea3550..f6686dce5ff 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java @@ -7,6 +7,7 @@ import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.JsonParser; import com.google.gson.JsonSyntaxException; import java.util.List; @@ -18,12 +19,13 @@ import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; /** - * json(value) Evaluates whether a string can be parsed as JSON format. Returns the string value if - * valid, null otherwise. Argument type: STRING Return type: STRING/NULL + * json(value) Evaluates whether the input can be parsed as JSON format. Returns the value if + * valid, null otherwise. Argument type: ANY Return type: ANY/NULL */ public class JsonFunctionImpl extends ImplementorUDF { public JsonFunctionImpl() { @@ -32,7 +34,7 @@ public JsonFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; + return ReturnTypes.ARG0_FORCE_NULLABLE; } public static class JsonImplementor implements NotNullImplementor { @@ -49,12 +51,13 @@ public Expression implement( public static Object eval(Object... args) { assert args.length == 1 : "Json only accept one argument"; - String value = (String) args[0]; + ObjectMapper mapper = new ObjectMapper(); + Object value = args[0]; try { - JsonParser.parseString(value); - return value; - } catch (JsonSyntaxException e) { - return null; + mapper.readTree(value.toString()); // try parse as JSON + return true; + } catch (Exception e) { + return false; } } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 04b72e635b3..aa8bd2b88df 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -76,25 +76,21 @@ public void testJsonArray() { verifyDataRows(actual, rows(List.of(1.0, 2.0, 0, -1.0, 1.1, -0.11))); } - @Ignore( - "We do throw the exception, but current way will resolve safe, so we will get Unsupported" - + " operator: json_array") @Test - public void testJsonArrayWithWrongType() { - IllegalArgumentException e = - assertThrows( - IllegalArgumentException.class, - () -> { - JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a = json_array(1, 2, 0, true, \"hello\", -0.11)| fields" - + " a | head 1", - TEST_INDEX_PEOPLE2)); - }); - verifyErrorMessageContains(e, "unsupported format"); + public void testJsonArrayWithDifferentType() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a = json_array(1, '123', json_object(\"name\", 3))| fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "array")); + + verifyDataRows(actual, rows(List.of(1, "123", Map.of("name", 3)))); } + + @Test public void testToJsonString() { JSONObject actual = From ea098e61d9dbae7273f44def0542db06f8758b99 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 15 May 2025 11:26:47 +0800 Subject: [PATCH 15/50] return string Signed-off-by: xinyual --- .../jsonUDF/JsonAppendFunctionImpl.java | 25 +++--- .../jsonUDF/JsonArrayFunctionImpl.java | 41 ++-------- .../jsonUDF/JsonDeleteFunctionImpl.java | 28 ++++--- .../jsonUDF/JsonExtendFunctionImpl.java | 17 ++-- .../jsonUDF/JsonExtractFunctionImpl.java | 50 ++++++++--- .../function/jsonUDF/JsonFunctionImpl.java | 12 +-- .../function/jsonUDF/JsonSetFunctionImpl.java | 27 +++--- .../function/jsonUDF/JsonUtils.java | 19 +++-- .../CalcitePPLJsonBuiltinFunctionIT.java | 82 +++++++++---------- .../opensearch/sql/ppl/JsonFunctionsIT.java | 19 ++--- 10 files changed, 152 insertions(+), 168 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java index a159aac959b..cc72798d12e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.function.jsonUDF; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; @@ -18,6 +19,8 @@ import com.jayway.jsonpath.PathNotFoundException; import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; + +import java.util.Arrays; import java.util.List; import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -40,12 +43,7 @@ public JsonAppendFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return opBinding -> { - RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); - return typeFactory.createMapType( - typeFactory.createSqlType(SqlTypeName.VARCHAR), - typeFactory.createSqlType(SqlTypeName.ANY)); - }; + return VARCHAR_FORCE_NULLABLE; } public static class JsonAppendImplementor implements NotNullImplementor { @@ -62,20 +60,17 @@ public Expression implement( public static Object eval(Object... args) throws JsonProcessingException { String jsonStr = (String) args[0]; - List keys = collectKeyValuePair(args); + List keys = Arrays.asList(args).subList(1, args.length); if (keys.size() % 2 != 0) { throw new RuntimeException( "Json append function needs corresponding path and values, but current get: " + keys); } - String resultStr = jsonAppendIfArray(jsonStr, keys, false); - Map result = gson.fromJson(resultStr, Map.class); - return result; + return jsonAppendIfArray(jsonStr, keys, false); } - public static String jsonAppendIfArray(String json, List pathValueMap, boolean isExtend) { - ObjectMapper mapper = new ObjectMapper(); + public static String jsonAppendIfArray(Object json, List pathValueMap, boolean isExtend) { try { - JsonNode tree = mapper.readTree(json); + JsonNode tree = verifyInput(json); Configuration conf = Configuration.builder() @@ -120,10 +115,10 @@ public static String jsonAppendIfArray(String json, List pathValueMap, b } } } - return mapper.writeValueAsString(tree); + return tree.toString(); } catch (Exception e) { if (e instanceof PathNotFoundException) { - return json; + return json.toString(); } throw new RuntimeException("Failed to process JSON", e); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java index 61bfab676d5..3907f2c39b4 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java @@ -7,18 +7,14 @@ import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.type.SqlReturnTypeInference; @@ -43,14 +39,10 @@ public JsonArrayFunctionImpl() { public SqlReturnTypeInference getReturnTypeInference() { return sqlOperatorBinding -> { RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - //List argTypes = sqlOperatorBinding.collectOperandTypes(); - //RelDataType commonType = typeFactory.leastRestrictive(argTypes); - //if (commonType == null) { - // throw new IllegalArgumentException( - // "All arguments in json array cannot be converted into one common types"); - //} return createArrayType( - typeFactory, typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.ANY), true), true); + typeFactory, + typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.ANY), true), + true); }; } @@ -58,36 +50,13 @@ public static class JsonArrayImplementor implements NotNullImplementor { @Override public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { - //RelDataType realType = call.getType().getComponentType(); - //List newArgs = new ArrayList<>(translatedOperands); - //assert realType != null; - //newArgs.add(Expressions.constant(realType.getSqlTypeName())); return Expressions.call( - Types.lookupMethod(JsonArrayFunctionImpl.class, "eval", Object[].class), translatedOperands); + Types.lookupMethod(JsonArrayFunctionImpl.class, "eval", Object[].class), + translatedOperands); } } public static Object eval(Object... args) { return Arrays.asList(args); - /* - SqlTypeName targetType = (SqlTypeName) args[args.length - 1]; - switch (targetType) { - case DOUBLE: - List unboxed = - IntStream.range(0, args.length - 1) - .mapToObj(i -> ((Number) args[i]).doubleValue()) - .collect(Collectors.toList()); - - return unboxed; - case FLOAT: - List unboxedFloat = - IntStream.range(0, args.length - 1) - .mapToObj(i -> ((Number) args[i]).floatValue()) - .collect(Collectors.toList()); - return unboxedFloat; - default: - return Arrays.asList(args).subList(0, args.length - 1); - } - */ } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java index 05d98c9b0a0..3fc9ea8bb02 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.function.jsonUDF; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; @@ -14,6 +15,8 @@ import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.PathNotFoundException; + +import java.util.Arrays; import java.util.List; import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -25,6 +28,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -36,12 +40,7 @@ public JsonDeleteFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return opBinding -> { - RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); - return typeFactory.createMapType( - typeFactory.createSqlType(SqlTypeName.VARCHAR), - typeFactory.createSqlType(SqlTypeName.ANY)); - }; + return VARCHAR_FORCE_NULLABLE; } public static class JsonDeleteImplementor implements NotNullImplementor { @@ -57,22 +56,25 @@ public Expression implement( } public static Object eval(Object... args) throws JsonProcessingException { - String jsonStr = (String) args[0]; - List jsonPaths = collectKeyValuePair(args); - ObjectMapper mapper = new ObjectMapper(); - JsonNode root = mapper.readTree(jsonStr); - DocumentContext ctx = JsonPath.parse(jsonStr); + List jsonPaths = Arrays.asList(args).subList(1, args.length); + JsonNode root = verifyInput(args[0]); + DocumentContext ctx; + if (args[0] instanceof String) { + ctx = JsonPath.parse(args[0].toString()); + } else { + ctx = JsonPath.parse(args[0]); + } for (Object originalPath : jsonPaths) { String jsonPath = convertToJsonPath(originalPath.toString()); try { Object matches = ctx.read(jsonPath); // verify whether it's a valid path } catch (PathNotFoundException e) { - return gson.fromJson(jsonStr, Map.class); + continue; } // Resolve path tokens PathTokenizer tokenizer = new PathTokenizer(jsonPath); root = deletePath(root, tokenizer); } - return gson.fromJson(mapper.writeValueAsString(root), Map.class); + return root.toString(); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java index e456942f572..698739142cd 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -5,11 +5,13 @@ package org.opensearch.sql.expression.function.jsonUDF; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; import static org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl.jsonAppendIfArray; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.collectKeyValuePair; import com.fasterxml.jackson.core.JsonProcessingException; + +import java.util.Arrays; import java.util.List; import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -32,12 +34,7 @@ public JsonExtendFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return opBinding -> { - RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); - return typeFactory.createMapType( - typeFactory.createSqlType(SqlTypeName.VARCHAR), - typeFactory.createSqlType(SqlTypeName.ANY)); - }; + return VARCHAR_FORCE_NULLABLE; } public static class JsonExtendImplementor implements NotNullImplementor { @@ -54,13 +51,11 @@ public Expression implement( public static Object eval(Object... args) throws JsonProcessingException { String jsonStr = (String) args[0]; - List keys = collectKeyValuePair(args); + List keys = Arrays.asList(args).subList(1, args.length); if (keys.size() % 2 != 0) { throw new RuntimeException( "Json extend function needs corresponding path and values, but current get: " + keys); } - String resultStr = jsonAppendIfArray(jsonStr, keys, true); - Map result = gson.fromJson(resultStr, Map.class); - return result; + return jsonAppendIfArray(jsonStr, keys, true); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index c7b30db1827..a94331888f7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -5,11 +5,15 @@ package org.opensearch.sql.expression.function.jsonUDF; +import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.convertToJsonPath; import com.jayway.jsonpath.JsonPath; + +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; @@ -17,9 +21,12 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; public class JsonExtractFunctionImpl extends ImplementorUDF { @@ -29,7 +36,18 @@ public JsonExtractFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; + return sqlOperatorBinding -> { + RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); + RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + if (sqlOperatorBinding.collectOperandTypes().size() > 2) { + return createArrayType( + typeFactory, + varcharType, + true); + } else { + return varcharType; + } + }; } public static class JsonExtractImplementor implements NotNullImplementor { @@ -48,15 +66,27 @@ public static Object eval(Object... args) { if (args.length < 2) { return null; } - String value = (String) args[0]; - String path = (String) args[1]; - path = convertToJsonPath(path); - try { - Object result = JsonPath.read(value, path); - result = result != null ? gson.toJson(result) : null; - return result; - } catch (Exception e) { - return null; + Object value = args[0]; + List results = new ArrayList<>(); + List paths = Arrays.asList(args).subList(1, args.length); + for (Object path: paths) { + String jsonPath = convertToJsonPath(path.toString()); + try { + Object result; + if (value instanceof String) { + result = JsonPath.read((String) value, jsonPath); + } else { + result = JsonPath.read(value, jsonPath); + } + result = result != null ? gson.toJson(result) : null; + results.add(result); + } catch (Exception e) { + results.add(null); + } + } + if (paths.size() > 1) { + return results; } + return results.get(0); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java index f6686dce5ff..13db90c2bc8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java @@ -5,11 +5,7 @@ package org.opensearch.sql.expression.function.jsonUDF; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; - import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.JsonParser; -import com.google.gson.JsonSyntaxException; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; @@ -24,8 +20,8 @@ import org.opensearch.sql.expression.function.ImplementorUDF; /** - * json(value) Evaluates whether the input can be parsed as JSON format. Returns the value if - * valid, null otherwise. Argument type: ANY Return type: ANY/NULL + * json(value) Evaluates whether the input can be parsed as JSON format. Returns the value if valid, + * null otherwise. Argument type: ANY Return type: ANY/NULL */ public class JsonFunctionImpl extends ImplementorUDF { public JsonFunctionImpl() { @@ -55,9 +51,9 @@ public static Object eval(Object... args) { Object value = args[0]; try { mapper.readTree(value.toString()); // try parse as JSON - return true; + return value; } catch (Exception e) { - return false; + return null; } } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java index 45328a72197..ad3ff928f9e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.function.jsonUDF; +import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; @@ -17,6 +18,8 @@ import com.jayway.jsonpath.PathNotFoundException; import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; + +import java.util.Arrays; import java.util.List; import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -39,12 +42,7 @@ public JsonSetFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return opBinding -> { - RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); - return typeFactory.createMapType( - typeFactory.createSqlType(SqlTypeName.VARCHAR), - typeFactory.createSqlType(SqlTypeName.ANY)); - }; + return VARCHAR_FORCE_NULLABLE; } public static class JsonSetImplementor implements NotNullImplementor { @@ -61,20 +59,17 @@ public Expression implement( public static Object eval(Object... args) { String jsonStr = (String) args[0]; - List keys = collectKeyValuePair(args); + List keys = Arrays.asList(args).subList(1, args.length); if (keys.size() % 2 != 0) { throw new RuntimeException( "Json set function needs corresponding path and values, but current get: " + keys); } - String resultStr = jsonSetIfParentObject(jsonStr, keys); - Map result = gson.fromJson(resultStr, Map.class); - return result; + return jsonSetIfParentObject(jsonStr, keys); } - public static String jsonSetIfParentObject(String json, List fullPathToValue) { - ObjectMapper mapper = new ObjectMapper(); + public static String jsonSetIfParentObject(Object json, List fullPathToValue) { try { - JsonNode root = mapper.readTree(json); + JsonNode root = verifyInput(json); Configuration conf = Configuration.builder() @@ -106,20 +101,20 @@ public static String jsonSetIfParentObject(String json, List fullPathToV if (JsonPath.isPathDefinite(parentPath)) { if (targets instanceof ObjectNode objectNode) { - objectNode.set(fieldName, mapper.valueToTree(value)); + objectNode.set(fieldName, objectMapper.valueToTree(value)); } } else { // Some * inside. an arrayNode returned for (int i = 0; i < targets.size(); i++) { JsonNode target = targets.get(i); if (target instanceof ObjectNode objectNode) { - objectNode.set(fieldName, mapper.valueToTree(value)); + objectNode.set(fieldName, objectMapper.valueToTree(value)); } } } } - return mapper.writeValueAsString(root); + return root.toString(); } catch (Exception e) { throw new RuntimeException("jsonSetIfParentObject failed", e); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java index 32be405e1b9..d24d441277b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java @@ -5,11 +5,14 @@ package org.opensearch.sql.expression.function.jsonUDF; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -287,11 +290,17 @@ public PathToken(String key, boolean isIndex) { } } - public static List collectKeyValuePair(Object... args) { - List result = new ArrayList<>(); - for (int i = 1; i < args.length; i++) { - result.add(args[i]); + public static JsonNode verifyInput(Object input) { + try { + JsonNode root; + if (input instanceof String) { + root = objectMapper.readTree(input.toString()); + } else { + root = objectMapper.valueToTree(input); + } + return root; + } catch (Exception e) { + throw new RuntimeException("fail to parse input", e); } - return result; } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index aa8bd2b88df..657906071ff 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -79,18 +79,17 @@ public void testJsonArray() { @Test public void testJsonArrayWithDifferentType() { JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a = json_array(1, '123', json_object(\"name\", 3))| fields a | head 1", - TEST_INDEX_PEOPLE2)); + executeQuery( + String.format( + "source=%s | eval a = json_array(1, '123', json_object(\"name\", 3))| fields a |" + + " head 1", + TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "array")); verifyDataRows(actual, rows(List.of(1, "123", Map.of("name", 3)))); } - - @Test public void testToJsonString() { JSONObject actual = @@ -102,7 +101,7 @@ public void testToJsonString() { verifySchema(actual, schema("a", "string")); - verifyDataRows(actual, rows("[1.0,2.0,0.0,-1.0,1.1,-0.11]")); + verifyDataRows(actual, rows("[1,2,0,-1,1.1,-0.11]")); } @Test @@ -238,9 +237,9 @@ public void testJsonSet() { + " fields a | head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "struct")); + verifySchema(actual, schema("a", "string")); - verifyDataRows(actual, rows(gson.fromJson("{\"a\":[{\"b\":\"3\"},{\"b\":\"3\"}]}", Map.class))); + verifyDataRows(actual, rows("{\"a\":[{\"b\":\"3\"},{\"b\":\"3\"}]}")); } @Test @@ -253,10 +252,10 @@ public void testJsonDelete() { + " 'age','gender')| fields a | head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "struct")); + verifySchema(actual, schema("a", "string")); verifyDataRows( - actual, rows(gson.fromJson("{\"account_number\":1,\"balance\":39225}", Map.class))); + actual, rows("{\"account_number\":1,\"balance\":39225}")); } @Test @@ -269,10 +268,10 @@ public void testJsonDeleteWithNested() { + " 'f2.f3') | fields a | head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "struct")); + verifySchema(actual, schema("a", "string")); verifyDataRows( - actual, rows(gson.fromJson("{\"f1\":\"abc\",\"f2\":{\"f4\":\"b\"}}", Map.class))); + actual, rows("{\"f1\":\"abc\",\"f2\":{\"f4\":\"b\"}}")); } @Test @@ -285,11 +284,11 @@ public void testJsonDeleteWithNestedNothing() { + " 'f2.f100') | fields a | head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "struct")); + verifySchema(actual, schema("a", "string")); verifyDataRows( actual, - rows(gson.fromJson("{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}", Map.class))); + rows("{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}")); } @Test @@ -302,12 +301,12 @@ public void testJsonDeleteWithNestedAndArray() { + " 'student{}.rank') | fields a | head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "struct")); + verifySchema(actual, schema("a", "string")); verifyDataRows( actual, rows( - gson.fromJson("{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}", Map.class))); + "{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}")); } @Test @@ -324,20 +323,20 @@ public void testJsonAppend() { + " 'school.teacher', json_array(\"Tom\", \"Walt\"))| fields a, b, c | head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "struct"), schema("b", "struct"), schema("c", "struct")); + verifySchema(actual, schema("a", "string"), schema("b", "string"), schema("c", "string")); verifyDataRows( actual, rows( - gson.fromJson( - "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tomy\",\"rank\":5}]}", - Map.class), - gson.fromJson( - "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}", - Map.class), - gson.fromJson( - "{\"school\":{\"teacher\":[\"Alice\",[\"Tom\",\"Walt\"]],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}", - Map.class))); + + "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tomy\",\"rank\":5}]}" + , + + "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}" + , + + "{\"school\":{\"teacher\":[\"Alice\",[\"Tom\",\"Walt\"]],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}" + )); } @Test @@ -354,30 +353,27 @@ public void testJsonExtend() { + " 'school.teacher', array(\"Tom\", \"Walt\"))| fields a, b, c | head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "struct"), schema("b", "struct"), schema("c", "struct")); + verifySchema(actual, schema("a", "string"), schema("b", "string"), schema("c", "string")); verifyDataRows( actual, rows( - gson.fromJson( - "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tommy\",\"rank\":5}]}", - Map.class), - gson.fromJson( - "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}", - Map.class), - gson.fromJson( - "{\"school\":{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}", - Map.class))); + "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tommy\",\"rank\":5}]}" + , + "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}" + , + "{\"school\":{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}" + )); } @Test public void test_cast_json() throws IOException { JSONObject result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" - + " | fields test_name", - TEST_INDEX_JSON_TEST)); - assertEquals(1, 1); + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" + + " | fields test_name", + TEST_INDEX_JSON_TEST)); + assertEquals(1, 1); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 08dc1651a47..f119f2e4694 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -11,11 +11,10 @@ import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; +import com.google.gson.Gson; import java.io.IOException; import java.util.List; import java.util.Map; - -import com.google.gson.Gson; import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; @@ -122,21 +121,19 @@ public void test_json() throws IOException { if (isCalciteEnabled()) { nestedArray = nestedArray.toString(); } - Object nestedObject = new JSONObject( - Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(Boolean.FALSE, 3))); + Object nestedObject = + new JSONObject(Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(Boolean.FALSE, 3))); if (isCalciteEnabled()) { - nestedObject = new Gson().toJson(Map.of("d", List.of(Boolean.FALSE, 3), "a", "1", "b", Map.of("c", "3"))); + nestedObject = + new Gson() + .toJson(Map.of("d", List.of(Boolean.FALSE, 3), "a", "1", "b", Map.of("c", "3"))); } verifyDataRows( result, - rows( - "json nested object", nestedObject - ), + rows("json nested object", nestedObject), rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))), rows("json array", new JSONArray(List.of(1, 2, 3, 4))), - rows( - "json nested array", - nestedArray), + rows("json nested array", nestedArray), rows("json scalar string", "abc"), rows("json scalar int", 1234), rows("json scalar float", 12.34), From 1de81560328f18d340bbcd2bf348e61e5573b36c Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 15 May 2025 11:50:51 +0800 Subject: [PATCH 16/50] apply spotless Signed-off-by: xinyual --- .../sql/calcite/CalcitePlanContext.java | 2 +- .../sql/calcite/CalciteRexNodeVisitor.java | 6 +- .../calcite/utils/BuiltinFunctionUtils.java | 643 ------------------ .../function/PPLBuiltinOperators.java | 6 +- .../jsonUDF/JsonAppendFunctionImpl.java | 10 +- .../jsonUDF/JsonArrayLengthFunctionImpl.java | 4 +- .../jsonUDF/JsonDeleteFunctionImpl.java | 11 +- .../jsonUDF/JsonExtendFunctionImpl.java | 9 +- .../jsonUDF/JsonExtractFunctionImpl.java | 17 +- .../jsonUDF/JsonKeysFunctionImpl.java | 2 +- .../function/jsonUDF/JsonSetFunctionImpl.java | 10 +- .../function/jsonUDF/JsonUtils.java | 159 +---- .../jsonUDF/ToJsonStringFunctionImpl.java | 6 +- .../remote/CalciteJsonFunctionsIT.java | 2 + .../standalone/CalciteArrayFunctionIT.java | 23 - .../CalcitePPLJsonBuiltinFunctionIT.java | 44 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 3 - 17 files changed, 47 insertions(+), 910 deletions(-) delete mode 100644 core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index ef4988c11b5..e2e1a3e5242 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -9,8 +9,8 @@ import java.sql.Connection; import java.util.HashMap; -import java.util.Map; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Stack; import java.util.function.BiFunction; diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index ae22ff12c31..4bc707f3337 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -12,18 +12,16 @@ import java.math.BigDecimal; import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import java.util.Collections; import java.util.List; +import java.util.Map; import lombok.RequiredArgsConstructor; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.sql.fun.SqlStdOperatorTable; diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java deleted file mode 100644 index 0c2f0a9092d..00000000000 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java +++ /dev/null @@ -1,643 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.calcite.utils; - -import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.*; - -import com.google.common.collect.ImmutableList; -import com.google.gson.Gson; -import java.math.BigDecimal; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.calcite.avatica.util.TimeUnit; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexLiteral; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.sql.SqlIntervalQualifier; -import org.apache.calcite.sql.SqlOperator; -import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.type.ReturnTypes; -import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql.type.SqlTypeTransforms; -import org.opensearch.sql.calcite.CalcitePlanContext; -import org.opensearch.sql.calcite.ExtendedRexBuilder; -import org.opensearch.sql.calcite.udf.conditionUDF.IfFunction; -import org.opensearch.sql.calcite.udf.conditionUDF.IfNullFunction; -import org.opensearch.sql.calcite.udf.conditionUDF.NullIfFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.ConvertTZFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.DateAddSubFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.DateDiffFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.DateFormatFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.DateFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.DatetimeFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.DayFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.DayOfWeekFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.DayOfYearFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.ExtractFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.FromDaysFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.FromUnixTimestampFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.GetFormatFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.HourFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.LastDayFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.MakeDateFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.MakeTimeFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.MicrosecondFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.MinuteFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.MinuteOfDayFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.MonthFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.PeriodAddFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.PeriodDiffFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.PeriodNameFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.PostprocessForUDTFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.QuarterFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.SecondFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.SecondToTimeFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.StrToDateFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.SysdateFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.TimeAddSubFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.TimeDiffFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.TimeFormatFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.TimeFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.TimeToSecondFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.TimestampAddFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.TimestampDiffFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.TimestampFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.ToDaysFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.ToSecondsFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.UnixTimeStampFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.UtcDateFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.UtcTimeFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.UtcTimeStampFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.WeekDayFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.WeekFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.YearFunction; -import org.opensearch.sql.calcite.udf.datetimeUDF.YearWeekFunction; -import org.opensearch.sql.calcite.udf.mathUDF.CRC32Function; -import org.opensearch.sql.calcite.udf.mathUDF.ConvFunction; -import org.opensearch.sql.calcite.udf.mathUDF.DivideFunction; -import org.opensearch.sql.calcite.udf.mathUDF.EulerFunction; -import org.opensearch.sql.calcite.udf.mathUDF.ModFunction; -import org.opensearch.sql.calcite.udf.mathUDF.SqrtFunction; -import org.opensearch.sql.calcite.udf.textUDF.LocateFunction; -import org.opensearch.sql.calcite.udf.textUDF.ReplaceFunction; - -/** - * TODO: We need to refactor code to make all related part together and directly return call TODO: - * Remove all makeFlag and use literal directly - */ -public interface BuiltinFunctionUtils { - - Set TIME_EXCLUSIVE_OPS = - Set.of("SECOND", "SECOND_OF_MINUTE", "MINUTE", "MINUTE_OF_HOUR", "HOUR", "HOUR_OF_DAY"); - - static SqlReturnTypeInference VARCHAR_FORCE_NULLABLE = - ReturnTypes.VARCHAR.andThen(SqlTypeTransforms.FORCE_NULLABLE); - static Gson gson = new Gson(); - - static SqlOperator translate(String op) { - String capitalOP = op.toUpperCase(Locale.ROOT); - switch (capitalOP) { - case "/": - return TransferUserDefinedFunction( - DivideFunction.class, "/", ReturnTypes.QUOTIENT_NULLABLE); - case "REPLACE": - return TransferUserDefinedFunction( - ReplaceFunction.class, "REPLACE", VARCHAR_FORCE_NULLABLE); - case "LOCATE": - return TransferUserDefinedFunction( - LocateFunction.class, - "LOCATE", - ReturnTypes.INTEGER.andThen(SqlTypeTransforms.FORCE_NULLABLE)); - case "CONV": - // The CONV function in PPL converts between numerical bases, - // while SqlStdOperatorTable.CONVERT converts between charsets. - return TransferUserDefinedFunction(ConvFunction.class, "CONVERT", VARCHAR_FORCE_NULLABLE); - case "CRC32": - return TransferUserDefinedFunction(CRC32Function.class, "CRC32", ReturnTypes.BIGINT); - case "E": - return TransferUserDefinedFunction(EulerFunction.class, "E", ReturnTypes.DOUBLE); - case "MOD", "%": - // The MOD function in PPL supports floating-point parameters, e.g., MOD(5.5, 2) = 1.5, - // MOD(3.1, 2.1) = 1.1, - // whereas SqlStdOperatorTable.MOD supports only integer / long parameters. - return TransferUserDefinedFunction(ModFunction.class, "MOD", ReturnTypes.LEAST_RESTRICTIVE); - case "SQRT": - // SqlStdOperatorTable.SQRT is declared but not implemented, therefore we use a custom - // implementation. - return TransferUserDefinedFunction( - SqrtFunction.class, "SQRT", ReturnTypes.DOUBLE_FORCE_NULLABLE); - // Built-in Date Functions - case "CURRENT_TIMESTAMP", "NOW", "LOCALTIMESTAMP", "LOCALTIME": - return TransferUserDefinedFunction( - PostprocessForUDTFunction.class, "POSTPROCESS", timestampInference); - case "CURTIME", "CURRENT_TIME": - return TransferUserDefinedFunction( - PostprocessForUDTFunction.class, "POSTPROCESS", timeInference); - case "CURRENT_DATE", "CURDATE": - return TransferUserDefinedFunction( - PostprocessForUDTFunction.class, "POSTPROCESS", dateInference); - case "DATE": - return TransferUserDefinedFunction(DateFunction.class, "DATE", dateInference); - case "DATE_ADD": - return TransferUserDefinedFunction( - DateAddSubFunction.class, "DATE_ADD", timestampInference); - case "ADDDATE": - return TransferUserDefinedFunction( - DateAddSubFunction.class, "ADDDATE", DateAddSubFunction.getReturnTypeForAddOrSubDate()); - case "SUBDATE": - return TransferUserDefinedFunction( - DateAddSubFunction.class, "SUBDATE", DateAddSubFunction.getReturnTypeForAddOrSubDate()); - case "DATE_SUB": - return TransferUserDefinedFunction( - DateAddSubFunction.class, "DATE_SUB", timestampInference); - case "ADDTIME", "SUBTIME": - return TransferUserDefinedFunction( - TimeAddSubFunction.class, capitalOP, TimeAddSubFunction.getReturnTypeForTimeAddSub()); - case "DAY_OF_WEEK", "DAYOFWEEK": - return TransferUserDefinedFunction( - DayOfWeekFunction.class, capitalOP, INTEGER_FORCE_NULLABLE); - case "DAY_OF_YEAR", "DAYOFYEAR": - return TransferUserDefinedFunction( - DayOfYearFunction.class, capitalOP, INTEGER_FORCE_NULLABLE); - case "EXTRACT": - // Reuse OpenSearch PPL's implementation - return TransferUserDefinedFunction(ExtractFunction.class, "EXTRACT", ReturnTypes.BIGINT); - case "CONVERT_TZ": - return TransferUserDefinedFunction( - ConvertTZFunction.class, "CONVERT_TZ", timestampInference); - case "DATETIME": - return TransferUserDefinedFunction(DatetimeFunction.class, "DATETIME", timestampInference); - - case "FROM_DAYS": - return TransferUserDefinedFunction(FromDaysFunction.class, "FROM_DAYS", dateInference); - case "DATE_FORMAT": - return TransferUserDefinedFunction( - DateFormatFunction.class, "DATE_FORMAT", VARCHAR_FORCE_NULLABLE); - case "GET_FORMAT": - return TransferUserDefinedFunction( - GetFormatFunction.class, "GET_FORMAT", VARCHAR_FORCE_NULLABLE); - case "MAKETIME": - return TransferUserDefinedFunction(MakeTimeFunction.class, "MAKETIME", timeInference); - case "MAKEDATE": - return TransferUserDefinedFunction(MakeDateFunction.class, "MAKEDATE", dateInference); - case "MINUTE_OF_DAY": - return TransferUserDefinedFunction( - MinuteOfDayFunction.class, "MINUTE_OF_DAY", ReturnTypes.INTEGER); - case "PERIOD_ADD": - return TransferUserDefinedFunction( - PeriodAddFunction.class, "PERIOD_ADD", ReturnTypes.INTEGER); - case "PERIOD_DIFF": - return TransferUserDefinedFunction( - PeriodDiffFunction.class, "PERIOD_DIFF", ReturnTypes.INTEGER); - case "STR_TO_DATE": - return TransferUserDefinedFunction( - StrToDateFunction.class, "STR_TO_DATE", timestampInference); - case "WEEK", "WEEK_OF_YEAR": - // WEEK in PPL support an additional mode argument, therefore we need to use a custom - // implementation. - return TransferUserDefinedFunction(WeekFunction.class, "WEEK", ReturnTypes.INTEGER); - // UDF Functions - // Built-in condition functions - case "IF": - return TransferUserDefinedFunction( - IfFunction.class, "if", ReturnTypes.ARG1.andThen(SqlTypeTransforms.FORCE_NULLABLE)); - case "IFNULL": - return TransferUserDefinedFunction( - IfNullFunction.class, "ifnull", ReturnTypes.ARG0_FORCE_NULLABLE); - case "NULLIF": - return TransferUserDefinedFunction( - NullIfFunction.class, "nullif", ReturnTypes.ARG0_FORCE_NULLABLE); - case "DAYNAME": - return TransferUserDefinedFunction( - PeriodNameFunction.class, "DAYNAME", VARCHAR_FORCE_NULLABLE); - case "MONTHNAME": - return TransferUserDefinedFunction( - PeriodNameFunction.class, "MONTHNAME", VARCHAR_FORCE_NULLABLE); - case "LAST_DAY": - return TransferUserDefinedFunction(LastDayFunction.class, "LAST_DAY", dateInference); - case "UNIX_TIMESTAMP": - return TransferUserDefinedFunction( - UnixTimeStampFunction.class, "unix_timestamp", ReturnTypes.DOUBLE); - case "SYSDATE": - return TransferUserDefinedFunction(SysdateFunction.class, "SYSDATE", timestampInference); - case "TIME": - return TransferUserDefinedFunction(TimeFunction.class, "TIME", timeInference); - case "TIMEDIFF": - return TransferUserDefinedFunction(TimeDiffFunction.class, "TIMEDIFF", timeInference); - case "TIME_TO_SEC": - return TransferUserDefinedFunction( - TimeToSecondFunction.class, "TIME_TO_SEC", ReturnTypes.BIGINT); - case "TIME_FORMAT": - return TransferUserDefinedFunction( - TimeFormatFunction.class, "TIME_FORMAT", VARCHAR_FORCE_NULLABLE); - case "TIMESTAMP": - // return SqlLibraryOperators.TIMESTAMP; - return TransferUserDefinedFunction( - TimestampFunction.class, "timestamp", timestampInference); - case "TIMESTAMPADD": - // return SqlLibraryOperators.TIMESTAMP; - return TransferUserDefinedFunction( - TimestampAddFunction.class, "TIMESTAMPADD", timestampInference); - case "TIMESTAMPDIFF": - return TransferUserDefinedFunction( - TimestampDiffFunction.class, "TIMESTAMPDIFF", ReturnTypes.BIGINT); - case "DATEDIFF": - return TransferUserDefinedFunction(DateDiffFunction.class, "DATEDIFF", ReturnTypes.BIGINT); - case "TO_SECONDS": - return TransferUserDefinedFunction( - ToSecondsFunction.class, "TO_SECONDS", ReturnTypes.BIGINT); - case "TO_DAYS": - return TransferUserDefinedFunction(ToDaysFunction.class, "TO_DAYS", ReturnTypes.BIGINT); - case "SEC_TO_TIME": - return TransferUserDefinedFunction( - SecondToTimeFunction.class, "SEC_TO_TIME", timeInference); - case "YEAR": - return TransferUserDefinedFunction(YearFunction.class, capitalOP, INTEGER_FORCE_NULLABLE); - case "QUARTER": - return TransferUserDefinedFunction( - QuarterFunction.class, capitalOP, INTEGER_FORCE_NULLABLE); - case "MINUTE", "MINUTE_OF_HOUR": - return TransferUserDefinedFunction(MinuteFunction.class, capitalOP, INTEGER_FORCE_NULLABLE); - case "HOUR", "HOUR_OF_DAY": - return TransferUserDefinedFunction(HourFunction.class, capitalOP, INTEGER_FORCE_NULLABLE); - case "MONTH", "MONTH_OF_YEAR": - return TransferUserDefinedFunction(MonthFunction.class, capitalOP, INTEGER_FORCE_NULLABLE); - case "DAY_OF_MONTH", "DAYOFMONTH", "DAY": - return TransferUserDefinedFunction(DayFunction.class, capitalOP, INTEGER_FORCE_NULLABLE); - case "SECOND", "SECOND_OF_MINUTE": - return TransferUserDefinedFunction(SecondFunction.class, capitalOP, INTEGER_FORCE_NULLABLE); - case "MICROSECOND": - return TransferUserDefinedFunction( - MicrosecondFunction.class, "MICROSECOND", ReturnTypes.INTEGER); - case "YEARWEEK": - return TransferUserDefinedFunction(YearWeekFunction.class, "YEARWEEK", ReturnTypes.INTEGER); - case "FROM_UNIXTIME": - return TransferUserDefinedFunction( - FromUnixTimestampFunction.class, - "FROM_UNIXTIME", - FromUnixTimestampFunction.interReturnTypes()); - case "WEEKDAY": - return TransferUserDefinedFunction(WeekDayFunction.class, "WEEKDAY", ReturnTypes.INTEGER); - case "UTC_TIMESTAMP": - return TransferUserDefinedFunction( - UtcTimeStampFunction.class, "utc_timestamp", timestampInference); - case "UTC_TIME": - return TransferUserDefinedFunction(UtcTimeFunction.class, "utc_time", timeInference); - case "UTC_DATE": - return TransferUserDefinedFunction(UtcDateFunction.class, "utc_date", dateInference); - // TODO Add more, ref RexImpTable - default: - throw new IllegalArgumentException("Unsupported operator: " + op); - } - } - - /** - * Translates function arguments to align with Calcite's expectations, ensuring compatibility with - * PPL (Piped Processing Language). This is necessary because Calcite's input argument order or - * default values may differ from PPL's function definitions. - * - * @param op The function name as a string. - * @param argList A list of {@link RexNode} representing the parsed arguments from the PPL - * statement. - * @param context The {@link CalcitePlanContext} providing necessary utilities such as {@code - * rexBuilder}. - * @return A modified list of {@link RexNode} that correctly maps to Calcite’s function - * expectations. - */ - static List translateArgument( - String op, List argList, CalcitePlanContext context, String currentTimestampStr) { - String capitalOP = op.toUpperCase(Locale.ROOT); - switch (capitalOP) { - case "DATE": - List dateArgs = - List.of( - argList.get(0), - context.rexBuilder.makeFlag(transferDateRelatedTimeName(argList.get(0))), - context.rexBuilder.makeLiteral(currentTimestampStr)); - return dateArgs; - case "HOUR", - "HOUR_OF_DAY", - "MINUTE", - "MINUTE_OF_HOUR", - "QUARTER", - "YEAR", - "LAST_DAY", - "DAY_OF_WEEK", - "DAYOFWEEK", - "DAY_OF_YEAR", - "DAYOFYEAR", - "MONTH", - "MONTH_OF_YEAR", - "DAY", - "DAY_OF_MONTH", - "DAYOFMONTH", - "SECOND", - "SECOND_OF_MINUTE": - return List.of( - argList.get(0), - context.rexBuilder.makeFlag(transferDateRelatedTimeName(argList.get(0))), - context.rexBuilder.makeLiteral(currentTimestampStr)); - case "CURRENT_TIMESTAMP", "NOW", "LOCALTIMESTAMP", "LOCALTIME": - RexNode currentTimestampCall = - context.rexBuilder.makeCall(SqlStdOperatorTable.CURRENT_TIMESTAMP, List.of()); - return List.of(currentTimestampCall, context.rexBuilder.makeFlag(SqlTypeName.TIMESTAMP)); - case "CURTIME", "CURRENT_TIME": - RexNode currentTimeCall = - context.rexBuilder.makeCall(SqlStdOperatorTable.CURRENT_TIME, List.of()); - return List.of(currentTimeCall, context.rexBuilder.makeFlag(SqlTypeName.TIME)); - case "CURRENT_DATE", "CURDATE": - RexNode currentDateCall = - context.rexBuilder.makeCall(SqlStdOperatorTable.CURRENT_DATE, List.of()); - return List.of(currentDateCall, context.rexBuilder.makeFlag(SqlTypeName.DATE)); - case "TIMESTAMP", - "TIMEDIFF", - "TIME_TO_SEC", - "TIME_FORMAT", - "TO_SECONDS", - "TO_DAYS", - "CONVERT_TZ": - List timestampArgs = new ArrayList<>(argList); - timestampArgs.addAll( - argList.stream() - .map(p -> context.rexBuilder.makeFlag(transferDateRelatedTimeName(p))) - .collect(Collectors.toList())); - timestampArgs.add(context.rexBuilder.makeLiteral(currentTimestampStr)); - return timestampArgs; - case "YEARWEEK", "WEEKDAY": - List weekdayArgs = new ArrayList<>(argList); - weekdayArgs.addAll( - argList.stream() - .map(p -> context.rexBuilder.makeFlag(transferDateRelatedTimeName(p))) - .collect(Collectors.toList())); - weekdayArgs.add(context.rexBuilder.makeLiteral(currentTimestampStr)); - return weekdayArgs; - case "TIMESTAMPADD": - List timestampAddArgs = new ArrayList<>(argList); - timestampAddArgs.add( - context.rexBuilder.makeFlag(argList.get(2).getType().getSqlTypeName())); - timestampAddArgs.add(context.rexBuilder.makeLiteral(currentTimestampStr)); - return timestampAddArgs; - case "TIMESTAMPDIFF": - List timestampDiffArgs = new ArrayList<>(); - timestampDiffArgs.add(argList.getFirst()); - timestampDiffArgs.addAll(buildArgsWithTypes(context.rexBuilder, argList, 1, 2)); - timestampDiffArgs.add(context.rexBuilder.makeLiteral(currentTimestampStr)); - return timestampDiffArgs; - case "DATEDIFF": - // datediff differs with timestamp diff in that it - List dateDiffArgs = buildArgsWithTypes(context.rexBuilder, argList, 0, 1); - dateDiffArgs.add(context.rexBuilder.makeLiteral(currentTimestampStr)); - return dateDiffArgs; - case "DAYNAME", "MONTHNAME": - List periodNameArgs = new ArrayList<>(); - periodNameArgs.add(argList.getFirst()); - periodNameArgs.add(context.rexBuilder.makeLiteral(capitalOP)); - periodNameArgs.add( - context.rexBuilder.makeFlag(argList.getFirst().getType().getSqlTypeName())); - return periodNameArgs; - case "DATE_SUB": - List dateSubArgs = transformDateManipulationArgs(argList, context.rexBuilder); - // A flag that represents isAdd - dateSubArgs.add(context.rexBuilder.makeLiteral(false)); - dateSubArgs.add(context.rexBuilder.makeFlag(SqlTypeName.TIMESTAMP)); - dateSubArgs.add(context.rexBuilder.makeLiteral(currentTimestampStr)); - return dateSubArgs; - case "DATE_ADD": - List dateAddArgs = transformDateManipulationArgs(argList, context.rexBuilder); - dateAddArgs.add(context.rexBuilder.makeLiteral(true)); - dateAddArgs.add(context.rexBuilder.makeFlag(SqlTypeName.TIMESTAMP)); - dateAddArgs.add(context.rexBuilder.makeLiteral(currentTimestampStr)); - return dateAddArgs; - case "ADDTIME": - SqlTypeName arg0Type = transferDateRelatedTimeName(argList.getFirst()); - SqlTypeName arg1Type = transferDateRelatedTimeName(argList.get(1)); - RexNode type0 = context.rexBuilder.makeFlag(arg0Type); - RexNode type1 = context.rexBuilder.makeFlag(arg1Type); - RexNode isAdd = context.rexBuilder.makeLiteral(true); - - return List.of( - argList.getFirst(), - type0, - argList.get(1), - type1, - isAdd, - context.rexBuilder.makeLiteral(currentTimestampStr)); - case "ADDDATE": - return transformAddOrSubDateArgs(argList, context.rexBuilder, true, currentTimestampStr); - case "SUBDATE": - return transformAddOrSubDateArgs(argList, context.rexBuilder, false, currentTimestampStr); - case "SUBTIME": - List subTimeArgs = transformTimeManipulationArgs(argList, context.rexBuilder); - subTimeArgs.add(context.rexBuilder.makeLiteral(false)); - subTimeArgs.add(context.rexBuilder.makeLiteral(currentTimestampStr)); - return subTimeArgs; - case "TIME": - return ImmutableList.of( - argList.getFirst(), - context.rexBuilder.makeFlag(transferDateRelatedTimeName(argList.getFirst()))); - case "DATE_FORMAT", "FORMAT_TIMESTAMP": - RexNode dateExpr = argList.get(0); - RexNode dateFormatPatternExpr = argList.get(1); - RexNode datetimeType; - datetimeType = context.rexBuilder.makeFlag(transferDateRelatedTimeName(dateExpr)); - return ImmutableList.of( - dateExpr, - datetimeType, - dateFormatPatternExpr, - context.rexBuilder.makeLiteral(currentTimestampStr)); - case "UNIX_TIMESTAMP": - List unixArgs = new ArrayList<>(argList); - unixArgs.add(context.rexBuilder.makeFlag(transferDateRelatedTimeName(argList.getFirst()))); - unixArgs.add(context.rexBuilder.makeLiteral(currentTimestampStr)); - return unixArgs; - case "WEEK", "WEEK_OF_YEAR": - RexNode woyMode; - if (argList.size() >= 2) { - woyMode = argList.get(1); - } else { - woyMode = - context.rexBuilder.makeLiteral( - 0, context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER)); - } - return List.of( - argList.getFirst(), - woyMode, - context.rexBuilder.makeFlag(argList.getFirst().getType().getSqlTypeName())); - case "STR_TO_DATE": - List strToDateArgs = new ArrayList<>(argList); - strToDateArgs.add(context.rexBuilder.makeLiteral(currentTimestampStr)); - return strToDateArgs; - case "MINUTE_OF_DAY", "MICROSECOND": - // Convert STRING/TIME/TIMESTAMP to TIMESTAMP - return ImmutableList.of( - argList.getFirst(), - context.rexBuilder.makeFlag(transferDateRelatedTimeName(argList.getFirst()))); - case "EXTRACT": - return ImmutableList.of( - argList.getFirst(), - argList.get(1), - context.rexBuilder.makeFlag(transferDateRelatedTimeName(argList.get(1))), - context.rexBuilder.makeLiteral(currentTimestampStr)); - case "DATETIME": - // Convert timestamp to a string to reuse OS PPL V2's implementation - RexNode argTimestamp = argList.getFirst(); - if (argTimestamp.getType().getSqlTypeName().equals(SqlTypeName.TIMESTAMP)) { - argTimestamp = - makeConversionCall( - "DATE_FORMAT", - ImmutableList.of(argTimestamp, context.rexBuilder.makeLiteral("%Y-%m-%d %T")), - context, - currentTimestampStr); - } - return Stream.concat(Stream.of(argTimestamp), argList.stream().skip(1)).toList(); - case "UTC_TIMESTAMP", "UTC_TIME", "UTC_DATE": - return List.of(context.rexBuilder.makeLiteral(currentTimestampStr)); - default: - return argList; - } - } - - private static RexNode makeConversionCall( - String funcName, - List arguments, - CalcitePlanContext context, - String currentTimestampStr) { - SqlOperator operator = translate(funcName); - List translatedArguments = - translateArgument(funcName, arguments, context, currentTimestampStr); - RelDataType returnType = - deriveReturnType(funcName, context.rexBuilder, operator, translatedArguments); - return context.rexBuilder.makeCall(returnType, operator, translatedArguments); - } - - static RelDataType deriveReturnType( - String funcName, RexBuilder rexBuilder, SqlOperator operator, List exprs) { - RelDataType returnType = - switch (funcName.toUpperCase(Locale.ROOT)) { - // This effectively invalidates the operand type check, which leads to unnecessary - // incompatible parameter type errors - case "DATEDIFF" -> rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT); - case "TIMESTAMPDIFF" -> rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT); - case "YEAR", - "MINUTE", - "HOUR", - "HOUR_OF_DAY", - "MONTH", - "MONTH_OF_YEAR", - "DAY_OF_YEAR", - "DAYOFYEAR", - "DAY_OF_MONTH", - "DAYOFMONTH", - "DAY_OF_WEEK", - "DAYOFWEEK", - "DAY", - "MINUTE_OF_HOUR", - "QUARTER", - "SECOND", - "SECOND_OF_MINUTE" -> rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER); - default -> rexBuilder.deriveReturnType(operator, exprs); - }; - // Make all return types nullable - return rexBuilder.getTypeFactory().createTypeWithNullability(returnType, true); - } - - private static List transformDateManipulationArgs( - List argList, ExtendedRexBuilder rexBuilder) { - List dateAddArgs = new ArrayList<>(); - RexNode baseTimestampExpr = argList.get(0); - RexNode intervalExpr = argList.get(1); - // 1. Add time unit - RexLiteral timeFrameName = - rexBuilder.makeFlag( - Objects.requireNonNull(intervalExpr.getType().getIntervalQualifier()).getUnit()); - dateAddArgs.add(timeFrameName); - // 2. Add interval - RexLiteral intervalArg = - rexBuilder.makeBigintLiteral(((RexLiteral) intervalExpr).getValueAs(BigDecimal.class)); - dateAddArgs.add(intervalArg); - // 3. Add timestamp - dateAddArgs.add(baseTimestampExpr); - // 4. Add original sql type - dateAddArgs.add(rexBuilder.makeFlag(transferDateRelatedTimeName(baseTimestampExpr))); - return dateAddArgs; - } - - private static List transformAddOrSubDateArgs( - List argList, - ExtendedRexBuilder rexBuilder, - Boolean isAdd, - String currentTimestampStr) { - List addOrSubDateArgs = new ArrayList<>(); - addOrSubDateArgs.add(argList.getFirst()); - SqlTypeName addType = argList.get(1).getType().getSqlTypeName(); - if (addType == SqlTypeName.BIGINT - || addType == SqlTypeName.DECIMAL - || addType == SqlTypeName.INTEGER) { - Number value = ((RexLiteral) argList.get(1)).getValueAs(Number.class); - addOrSubDateArgs.add( - rexBuilder.makeIntervalLiteral( - new BigDecimal(value.toString()), - new SqlIntervalQualifier(TimeUnit.DAY, null, SqlParserPos.ZERO))); - } else { - addOrSubDateArgs.add(argList.get(1)); - } - List addOrSubDateRealInput = - transformDateManipulationArgs(addOrSubDateArgs, rexBuilder); - addOrSubDateRealInput.add(rexBuilder.makeLiteral(isAdd)); - SqlTypeName firstType = transferDateRelatedTimeName(argList.getFirst()); - if (firstType == SqlTypeName.DATE - && (addType == SqlTypeName.BIGINT - || addType == SqlTypeName.DECIMAL - || addType == SqlTypeName.INTEGER)) { - addOrSubDateRealInput.add(rexBuilder.makeFlag(SqlTypeName.DATE)); - addOrSubDateRealInput.add( - rexBuilder.makeLiteral(0, rexBuilder.getTypeFactory().createSqlType(SqlTypeName.DATE))); - } else { - addOrSubDateRealInput.add(rexBuilder.makeFlag(SqlTypeName.TIMESTAMP)); - addOrSubDateRealInput.add( - rexBuilder.makeLiteral( - 0L, rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TIMESTAMP))); - } - addOrSubDateRealInput.add(rexBuilder.makeLiteral(currentTimestampStr)); - return addOrSubDateRealInput; - } - - private static List transformTimeManipulationArgs( - List argList, ExtendedRexBuilder rexBuilder) { - SqlTypeName arg0Type = transferDateRelatedTimeName(argList.getFirst()); - SqlTypeName arg1Type = transferDateRelatedTimeName(argList.get(1)); - RexNode type0 = rexBuilder.makeFlag(arg0Type); - RexNode type1 = rexBuilder.makeFlag(arg1Type); - return new ArrayList<>(List.of(argList.getFirst(), type0, argList.get(1), type1)); - } - - /** - * Builds a list of RexNodes where each selected argument is followed by a RexNode representing - * its SQL type. - * - * @param rexBuilder the RexBuilder instance used to create type flags - * @param args the original list of arguments - * @return A new list of RexNodes: [arg, typeFlag, arg, typeFlag, ...] - */ - private static List buildArgsWithTypes( - RexBuilder rexBuilder, List args, int... indexes) { - List result = new ArrayList<>(); - for (int index : indexes) { - RexNode arg = args.get(index); - result.add(arg); - result.add(rexBuilder.makeFlag(transferDateRelatedTimeName(arg))); - } - return result; - } -} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 04605929da8..8f261f1a64a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -23,6 +23,9 @@ import org.apache.calcite.sql.type.SqlTypeTransforms; import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable; import org.apache.calcite.util.BuiltInMethod; +import org.opensearch.sql.calcite.utils.PPLReturnTypes; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.expression.datetime.DateTimeFunctions; import org.opensearch.sql.expression.function.collectionUDF.ForallFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonArrayFunctionImpl; @@ -36,9 +39,6 @@ import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonValidFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.ToJsonStringFunctionImpl; -import org.opensearch.sql.calcite.utils.PPLReturnTypes; -import org.opensearch.sql.data.type.ExprCoreType; -import org.opensearch.sql.expression.datetime.DateTimeFunctions; import org.opensearch.sql.expression.function.udf.CryptographicFunction; import org.opensearch.sql.expression.function.udf.datetime.AddSubDateFunction; import org.opensearch.sql.expression.function.udf.datetime.CurrentFunction; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java index cc72798d12e..f966bc525d8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -5,13 +5,11 @@ package org.opensearch.sql.expression.function.jsonUDF; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.DocumentContext; @@ -19,21 +17,17 @@ import com.jayway.jsonpath.PathNotFoundException; import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; - import java.util.Arrays; import java.util.List; -import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; public class JsonAppendFunctionImpl extends ImplementorUDF { @@ -43,7 +37,7 @@ public JsonAppendFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; + return STRING_FORCE_NULLABLE; } public static class JsonAppendImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java index ddbcae4de44..b1281a4fa5b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java @@ -5,8 +5,8 @@ package org.opensearch.sql.expression.function.jsonUDF; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; -import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.INTEGER_FORCE_NULLABLE; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.INTEGER_FORCE_NULLABLE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; import com.google.gson.JsonSyntaxException; import java.util.List; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java index 3fc9ea8bb02..b905831fc2d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java @@ -5,32 +5,25 @@ package org.opensearch.sql.expression.function.jsonUDF; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.PathNotFoundException; - import java.util.Arrays; import java.util.List; -import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; -import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; public class JsonDeleteFunctionImpl extends ImplementorUDF { @@ -40,7 +33,7 @@ public JsonDeleteFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; + return STRING_FORCE_NULLABLE; } public static class JsonDeleteImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java index 698739142cd..1c1f6970cb8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -5,26 +5,21 @@ package org.opensearch.sql.expression.function.jsonUDF; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; import static org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl.jsonAppendIfArray; import com.fasterxml.jackson.core.JsonProcessingException; - import java.util.Arrays; import java.util.List; -import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; public class JsonExtendFunctionImpl extends ImplementorUDF { @@ -34,7 +29,7 @@ public JsonExtendFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; + return STRING_FORCE_NULLABLE; } public static class JsonExtendImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index a94331888f7..c7949ae9055 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -6,12 +6,10 @@ package org.opensearch.sql.expression.function.jsonUDF; import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.convertToJsonPath; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; import com.jayway.jsonpath.JsonPath; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -38,13 +36,12 @@ public JsonExtractFunctionImpl() { public SqlReturnTypeInference getReturnTypeInference() { return sqlOperatorBinding -> { RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - RelDataType varcharType = typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.VARCHAR), true); + RelDataType varcharType = + typeFactory.createTypeWithNullability( + typeFactory.createSqlType(SqlTypeName.VARCHAR), true); if (sqlOperatorBinding.collectOperandTypes().size() > 2) { - return createArrayType( - typeFactory, - varcharType, - true); - } else { + return createArrayType(typeFactory, varcharType, true); + } else { return varcharType; } }; @@ -69,7 +66,7 @@ public static Object eval(Object... args) { Object value = args[0]; List results = new ArrayList<>(); List paths = Arrays.asList(args).subList(1, args.length); - for (Object path: paths) { + for (Object path : paths) { String jsonPath = convertToJsonPath(path.toString()); try { Object result; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java index 82959779826..7073bfead84 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java @@ -6,7 +6,7 @@ package org.opensearch.sql.expression.function.jsonUDF; import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; import com.google.gson.JsonSyntaxException; import java.util.Arrays; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java index ad3ff928f9e..b5d9e6d3d9d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -5,12 +5,10 @@ package org.opensearch.sql.expression.function.jsonUDF; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.DocumentContext; @@ -18,21 +16,17 @@ import com.jayway.jsonpath.PathNotFoundException; import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; - import java.util.Arrays; import java.util.List; -import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; public class JsonSetFunctionImpl extends ImplementorUDF { @@ -42,7 +36,7 @@ public JsonSetFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; + return STRING_FORCE_NULLABLE; } public static class JsonSetImplementor implements NotNullImplementor { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java index d24d441277b..04f8ba64245 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java @@ -5,168 +5,17 @@ package org.opensearch.sql.expression.function.jsonUDF; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.gson.Gson; import java.util.ArrayList; -import java.util.Arrays; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; public class JsonUtils { static ObjectMapper objectMapper = new ObjectMapper(); - - static Object parseValue(String value) { - // Try parsing the value as JSON, fallback to primitive if parsing fails - try { - return objectMapper.readValue(value, Object.class); - } catch (Exception e) { - // Primitive value, return as is - return value; - } - } - - @FunctionalInterface - interface UpdateConsumer { - void apply(Map obj, String key, Object value); - } - - private static void traverseNestedObject( - Object currentObj, - String[] pathParts, - int depth, - Object valueToUpdate, - UpdateConsumer updateObjectFunction) { - if (currentObj == null || depth >= pathParts.length) { - return; - } - - if (currentObj instanceof Map) { - Map currentMap = (Map) currentObj; - String currentKey = pathParts[depth]; - - if (depth == pathParts.length - 1) { - updateObjectFunction.apply(currentMap, currentKey, valueToUpdate); - } else { - // Continue traversing - currentMap.computeIfAbsent( - currentKey, k -> new LinkedHashMap<>()); // Create map if not present - traverseNestedObject( - currentMap.get(currentKey), pathParts, depth + 1, valueToUpdate, updateObjectFunction); - } - } else if (currentObj instanceof List) { - // If the current object is a list, process each map in the list - List list = (List) currentObj; - for (Object item : list) { - if (item instanceof Map) { - traverseNestedObject(item, pathParts, depth, valueToUpdate, updateObjectFunction); - } - } - } - } - - static String updateNestedJson( - String jsonStr, List pathValues, UpdateConsumer updateFieldConsumer) { - if (jsonStr == null) { - return null; - } - // don't update if the list is empty, or the list is not key-value pairs - if (pathValues.isEmpty()) { - return jsonStr; - } - try { - // Parse the JSON string into a Map - Map jsonMap = objectMapper.readValue(jsonStr, Map.class); - - // Iterate through the key-value pairs and update the json - var iter = pathValues.iterator(); - while (iter.hasNext()) { - String path = iter.next(); - if (!iter.hasNext()) { - // no value provided and cannot update anything - break; - } - String[] pathParts = path.split("\\."); - Object parsedValue = parseValue(iter.next()); - - traverseNestedObject(jsonMap, pathParts, 0, parsedValue, updateFieldConsumer); - } - - // Convert the updated map back to JSON - return objectMapper.writeValueAsString(jsonMap); - } catch (Exception e) { - return null; - } - } - - static void appendObjectValue(Map obj, String key, Object value) { - // If it's the last key, append to the array - obj.computeIfAbsent(key, k -> new ArrayList<>()); // Create list if not present - Object existingValue = obj.get(key); - - if (existingValue instanceof List) { - List list = (List) existingValue; - list.add(value); - } - } - - static void extendObjectValue(Map obj, String key, Object value) { - // If it's the last key, append to the array - obj.computeIfAbsent(key, k -> new ArrayList<>()); // Create list if not present - Object existingValue = obj.get(key); - - if (existingValue instanceof List) { - List existingList = (List) existingValue; - if (value instanceof List) { - existingList.addAll((List) value); - } else { - existingList.add(value); - } - } - } - - /** - * remove nested json object using its keys parts. - * - * @param currentObj - * @param keyParts - * @param depth - */ - static void removeNestedKey(Object currentObj, String[] keyParts, int depth) { - if (currentObj == null || depth >= keyParts.length) { - return; - } - - if (currentObj instanceof Map) { - Map currentMap = (Map) currentObj; - String currentKey = keyParts[depth]; - - if (depth == keyParts.length - 1) { - // If it's the last key, remove it from the map - currentMap.remove(currentKey); - } else { - // If not the last key, continue traversing - if (currentMap.containsKey(currentKey)) { - Object nextObj = currentMap.get(currentKey); - - if (nextObj instanceof List) { - // If the value is a list, process each item in the list - List list = (List) nextObj; - for (int i = 0; i < list.size(); i++) { - removeNestedKey(list.get(i), keyParts, depth + 1); - } - } else { - // Continue traversing if it's a map - removeNestedKey(nextObj, keyParts, depth + 1); - } - } - } - } - } + static Gson gson = new Gson(); /** * @param input candidate json path like a.b{}.c{2} @@ -294,13 +143,13 @@ public static JsonNode verifyInput(Object input) { try { JsonNode root; if (input instanceof String) { - root = objectMapper.readTree(input.toString()); + root = objectMapper.readTree(input.toString()); } else { root = objectMapper.valueToTree(input); } return root; } catch (Exception e) { - throw new RuntimeException("fail to parse input", e); + throw new RuntimeException("fail to parse input", e); } } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java index 7de40e109ba..e751b3c34a8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java @@ -5,8 +5,8 @@ package org.opensearch.sql.expression.function.jsonUDF; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.VARCHAR_FORCE_NULLABLE; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -27,7 +27,7 @@ public ToJsonStringFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return VARCHAR_FORCE_NULLABLE; + return STRING_FORCE_NULLABLE; } public static class ToJsonStringImplementor implements NotNullImplementor { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteJsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteJsonFunctionsIT.java index 6e0af4881cf..06493982016 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteJsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteJsonFunctionsIT.java @@ -5,8 +5,10 @@ package org.opensearch.sql.calcite.remote; +import org.junit.Ignore; import org.opensearch.sql.ppl.JsonFunctionsIT; +@Ignore public class CalciteJsonFunctionsIT extends JsonFunctionsIT { @Override public void init() throws Exception { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java index 6b08f1e7695..6fc03c846ad 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java @@ -5,15 +5,9 @@ package org.opensearch.sql.calcite.standalone; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.util.MatcherUtils.*; -import static org.opensearch.sql.util.MatcherUtils.rows; import java.io.IOException; -import java.util.Map; -import org.json.JSONObject; -import org.junit.jupiter.api.Test; public class CalciteArrayFunctionIT extends CalcitePPLIntegTestCase { @Override @@ -27,21 +21,4 @@ public void init() throws IOException { loadIndex(Index.PEOPLE2); loadIndex(Index.BANK); } - - @Test - public void testForAll() { - JSONObject actual = - executeQuery( - String.format( - "source=%s | eval array = array(1, -1, 2), result = forall(array, x -> x > 0) |" - + " fields result | head 1", - TEST_INDEX_BANK)); - - verifySchema(actual, schema("a", "struct")); - - verifyDataRows( - actual, - rows( - gson.fromJson("{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}", Map.class))); - } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 657906071ff..25cce34d940 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -5,11 +5,11 @@ package org.opensearch.sql.calcite.standalone; -import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.gson; import static org.opensearch.sql.legacy.TestsConstants.*; import static org.opensearch.sql.util.MatcherUtils.*; import static org.opensearch.sql.util.MatcherUtils.rows; +import com.google.gson.Gson; import java.io.IOException; import java.util.List; import java.util.Map; @@ -59,8 +59,8 @@ public void testJsonObject() { verifyDataRows( actual, rows( - gson.fromJson("{\"key\":123.45}", Map.class), - gson.fromJson("{\"outer\":{\"inner\":123.45}}", Map.class))); + new Gson().fromJson("{\"key\":123.45}", Map.class), + new Gson().fromJson("{\"outer\":{\"inner\":123.45}}", Map.class))); } @Test @@ -181,7 +181,7 @@ public void testJsonExtractWithNewPath() { verifyDataRows( actual, rows( - gson.toJson(gson.fromJson(candidate, List.class)), + new Gson().toJson(new Gson().fromJson(candidate, List.class)), "8981.0", "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0}")); } @@ -254,8 +254,7 @@ public void testJsonDelete() { verifySchema(actual, schema("a", "string")); - verifyDataRows( - actual, rows("{\"account_number\":1,\"balance\":39225}")); + verifyDataRows(actual, rows("{\"account_number\":1,\"balance\":39225}")); } @Test @@ -270,8 +269,7 @@ public void testJsonDeleteWithNested() { verifySchema(actual, schema("a", "string")); - verifyDataRows( - actual, rows("{\"f1\":\"abc\",\"f2\":{\"f4\":\"b\"}}")); + verifyDataRows(actual, rows("{\"f1\":\"abc\",\"f2\":{\"f4\":\"b\"}}")); } @Test @@ -286,9 +284,7 @@ public void testJsonDeleteWithNestedNothing() { verifySchema(actual, schema("a", "string")); - verifyDataRows( - actual, - rows("{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}")); + verifyDataRows(actual, rows("{\"f1\":\"abc\",\"f2\":{\"f3\":\"a\",\"f4\":\"b\"}}")); } @Test @@ -303,10 +299,7 @@ public void testJsonDeleteWithNestedAndArray() { verifySchema(actual, schema("a", "string")); - verifyDataRows( - actual, - rows( - "{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}")); + verifyDataRows(actual, rows("{\"student\":[{\"name\":\"Bob\"},{\"name\":\"Charlie\"}]}")); } @Test @@ -328,15 +321,9 @@ public void testJsonAppend() { verifyDataRows( actual, rows( - - "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tomy\",\"rank\":5}]}" - , - - "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}" - , - - "{\"school\":{\"teacher\":[\"Alice\",[\"Tom\",\"Walt\"]],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}" - )); + "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tomy\",\"rank\":5}]}", + "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}", + "{\"school\":{\"teacher\":[\"Alice\",[\"Tom\",\"Walt\"]],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}")); } @Test @@ -358,12 +345,9 @@ public void testJsonExtend() { verifyDataRows( actual, rows( - "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tommy\",\"rank\":5}]}" - , - "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}" - , - "{\"school\":{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}" - )); + "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tommy\",\"rank\":5}]}", + "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}", + "{\"school\":{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}")); } @Test diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index a904a66427e..352ebfb026e 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -702,7 +702,6 @@ trigonometricFunctionName | TAN ; -<<<<<<< HEAD collectionFunctionName : ARRAY ; @@ -729,14 +728,12 @@ lambdaFunctionName | REDUCE ; -======= cryptographicFunctionName : MD5 | SHA1 | SHA2 ; ->>>>>>> origin/main dateTimeFunctionName : ADDDATE | ADDTIME From 3fc8f384dd9a816ae0f90b42a72b81f78bb7063d Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 15 May 2025 12:40:01 +0800 Subject: [PATCH 17/50] revert useless change Signed-off-by: xinyual --- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 7 ------- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 13 +------------ .../org/opensearch/sql/ppl/parser/AstBuilder.java | 2 +- .../sql/ppl/parser/AstExpressionBuilder.java | 11 ----------- 4 files changed, 2 insertions(+), 31 deletions(-) diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 00afc88e563..13b5b1f65b0 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -385,13 +385,6 @@ JSON_DELETE: 'JSON_DELETE'; JSON_APPEND: 'JSON_APPEND'; JSON_EXTEND: 'JSON_EXTEND'; -// LAMBDA FUNCTIONS -//EXISTS: 'EXISTS'; -FORALL: 'FORALL'; -FILTER: 'FILTER'; -TRANSFORM: 'TRANSFORM'; -REDUCE: 'REDUCE'; - // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; NULLIF: 'NULLIF'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 352ebfb026e..2a90c4b89ce 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -462,8 +462,6 @@ valueExpression | timestampFunction # timestampFunctionCall | LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr | LT_SQR_PRTHS subSearch RT_SQR_PRTHS # scalarSubqueryExpr - | ident ARROW expression # lambda - | LT_PRTHS ident (COMMA ident)+ RT_PRTHS ARROW expression # lambda ; primaryExpression @@ -583,7 +581,6 @@ evalFunctionName | jsonFunctionName | geoipFunctionName | collectionFunctionName - | lambdaFunctionName ; functionArgs @@ -591,7 +588,7 @@ functionArgs ; functionArg - : (ident EQUAL)? valueExpression + : (ident EQUAL)? expression ; relevanceArg @@ -720,14 +717,6 @@ jsonFunctionName | JSON_EXTEND ; -lambdaFunctionName - : FORALL - | EXISTS - | FILTER - | TRANSFORM - | REDUCE - ; - cryptographicFunctionName : MD5 | SHA1 diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 397396d45a5..adde04f2b1f 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -562,7 +562,7 @@ public UnresolvedPlan visitTableFunction(TableFunctionContext ctx) { String argName = (arg.ident() != null) ? arg.ident().getText() : null; builder.add( new UnresolvedArgument( - argName, this.internalVisitExpression(arg.valueExpression()))); + argName, this.internalVisitExpression(arg.expression()))); }); return new TableFunction(this.internalVisitExpression(ctx.qualifiedName()), builder.build()); } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index b8da1a14ed8..f7629a2d1a9 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -133,17 +133,6 @@ public UnresolvedExpression visitLogicalXor(LogicalXorContext ctx) { return new Xor(visit(ctx.left), visit(ctx.right)); } - @Override - public UnresolvedExpression visitLambda(OpenSearchPPLParser.LambdaContext ctx) { - - List arguments = - ctx.ident().stream() - .map(x -> this.visitIdentifiers(Collections.singletonList(x))) - .collect(Collectors.toList()); - UnresolvedExpression function = visitExpression(ctx.expression()); - return new LambdaFunction(function, arguments); - } - /** Comparison expression. */ @Override public UnresolvedExpression visitCompareExpr(CompareExprContext ctx) { From 34a616f0f78213dc7a0833c11e8ebaf6ec1a1280 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 15 May 2025 12:40:13 +0800 Subject: [PATCH 18/50] apply spotless Signed-off-by: xinyual --- .../main/java/org/opensearch/sql/ppl/parser/AstBuilder.java | 3 +-- .../org/opensearch/sql/ppl/parser/AstExpressionBuilder.java | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index adde04f2b1f..5d1bfc97df0 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -561,8 +561,7 @@ public UnresolvedPlan visitTableFunction(TableFunctionContext ctx) { arg -> { String argName = (arg.ident() != null) ? arg.ident().getText() : null; builder.add( - new UnresolvedArgument( - argName, this.internalVisitExpression(arg.expression()))); + new UnresolvedArgument(argName, this.internalVisitExpression(arg.expression()))); }); return new TableFunction(this.internalVisitExpression(ctx.qualifiedName()), builder.build()); } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index f7629a2d1a9..679565e22f6 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -44,7 +44,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; From 1202b87f3e0abda12660fb2d7d9fd148d3440aec Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 15 May 2025 12:40:36 +0800 Subject: [PATCH 19/50] revert useless change Signed-off-by: xinyual --- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 - 1 file changed, 1 deletion(-) diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 13b5b1f65b0..88536d4f2eb 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -192,7 +192,6 @@ RT_SQR_PRTHS: ']'; SINGLE_QUOTE: '\''; DOUBLE_QUOTE: '"'; BACKTICK: '`'; -ARROW: '->'; // Operators. Bit From 7690d78f949eae6870b9b6b8b87a65fec58fad79 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 15 May 2025 12:41:27 +0800 Subject: [PATCH 20/50] revert useless change Signed-off-by: xinyual --- .../standalone/CalciteArrayFunctionIT.java | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java deleted file mode 100644 index 6fc03c846ad..00000000000 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalciteArrayFunctionIT.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.calcite.standalone; - -import static org.opensearch.sql.util.MatcherUtils.*; - -import java.io.IOException; - -public class CalciteArrayFunctionIT extends CalcitePPLIntegTestCase { - @Override - public void init() throws IOException { - super.init(); - loadIndex(Index.STATE_COUNTRY); - loadIndex(Index.STATE_COUNTRY_WITH_NULL); - loadIndex(Index.DATE_FORMATS); - loadIndex(Index.BANK_WITH_NULL_VALUES); - loadIndex(Index.DATE); - loadIndex(Index.PEOPLE2); - loadIndex(Index.BANK); - } -} From eed5aa9ad9dcafa2a7e371ac796ef0fa62205421 Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 16 May 2025 13:21:09 +0800 Subject: [PATCH 21/50] align with splunk Signed-off-by: xinyual --- .../expression/function/PPLFuncImpTable.java | 37 ++++- .../jsonUDF/JsonAppendFunctionImpl.java | 16 +- .../jsonUDF/JsonArrayFunctionImpl.java | 17 +- .../jsonUDF/JsonDeleteFunctionImpl.java | 30 +--- .../jsonUDF/JsonExtendFunctionImpl.java | 37 ++++- .../jsonUDF/JsonExtractFunctionImpl.java | 97 +++++++----- .../function/jsonUDF/JsonFunctionImpl.java | 12 +- .../jsonUDF/JsonKeysFunctionImpl.java | 27 +--- .../jsonUDF/JsonObjectFunctionImpl.java | 21 +-- .../function/jsonUDF/JsonSetFunctionImpl.java | 73 ++------- .../function/jsonUDF/JsonUtils.java | 149 ++++++++---------- .../jsonUDF/ToJsonStringFunctionImpl.java | 8 +- integ-test/build.gradle | 4 +- .../CalcitePPLJsonBuiltinFunctionIT.java | 59 +++---- 14 files changed, 277 insertions(+), 310 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 1ffb1bb593f..9e9e6372d1e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.function; +import static org.apache.calcite.sql.SqlJsonConstructorNullClause.NULL_ON_NULL; import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.getLegacyTypeName; import static org.opensearch.sql.expression.function.BuiltinFunctionName.*; @@ -16,6 +17,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Stream; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; @@ -82,6 +84,23 @@ default List getParams() { } } + public interface FunctionImpAny extends FunctionImp { + RexNode resolve(RexBuilder builder, List additonalArgs, RexNode... args); + + @Override + default RexNode resolve(RexBuilder builder, RexNode... args) { + if (args.length != 2) { + throw new IllegalArgumentException("This function requires exactly 2 arguments"); + } + return resolve(builder, args[0], args[1]); + } + + @Override + default List getParams() { + return null; + } + } + /** The singleton instance. */ public static final PPLFuncImpTable INSTANCE; @@ -305,14 +324,26 @@ void populate() { registerOperator(ARRAY, PPLBuiltinOperators.ARRAY); // Register Json function + register( + JSON_ARRAY, + ((builder, args) -> + builder.makeCall( + SqlStdOperatorTable.JSON_ARRAY, + Stream.concat(Stream.of(builder.makeFlag(NULL_ON_NULL)), Arrays.stream(args)) + .toArray(RexNode[]::new)))); + register( + JSON_OBJECT, + ((builder, args) -> + builder.makeCall( + SqlStdOperatorTable.JSON_OBJECT, + Stream.concat(Stream.of(builder.makeFlag(NULL_ON_NULL)), Arrays.stream(args)) + .toArray(RexNode[]::new)))); registerOperator(JSON, PPLBuiltinOperators.JSON); - registerOperator(JSON_OBJECT, PPLBuiltinOperators.JSON_OBJECT); - registerOperator(JSON_ARRAY, PPLBuiltinOperators.JSON_ARRAY); registerOperator(TO_JSON_STRING, PPLBuiltinOperators.TO_JSON_STRING); registerOperator(JSON_ARRAY_LENGTH, PPLBuiltinOperators.JSON_ARRAY_LENGTH); registerOperator(JSON_EXTRACT, PPLBuiltinOperators.JSON_EXTRACT); registerOperator(JSON_KEYS, PPLBuiltinOperators.JSON_KEYS); - registerOperator(JSON_VALID, PPLBuiltinOperators.JSON_VALID); + registerOperator(JSON_VALID, SqlStdOperatorTable.IS_JSON_VALUE); registerOperator(JSON_SET, PPLBuiltinOperators.JSON_SET); registerOperator(JSON_DELETE, PPLBuiltinOperators.JSON_DELETE); registerOperator(JSON_APPEND, PPLBuiltinOperators.JSON_APPEND); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java index f966bc525d8..4122d12bffe 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -17,6 +17,7 @@ import com.jayway.jsonpath.PathNotFoundException; import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -26,6 +27,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -59,7 +61,19 @@ public static Object eval(Object... args) throws JsonProcessingException { throw new RuntimeException( "Json append function needs corresponding path and values, but current get: " + keys); } - return jsonAppendIfArray(jsonStr, keys, false); + JsonNode root = verifyInput(args[0]); + List expands = new ArrayList<>(); + for (int i = 0; i < keys.size(); i += 2) { + List expandedPaths = expandJsonPath(root, convertToJsonPath(keys.get(i).toString())); + for (String expandedPath : expandedPaths) { + expands.add( + expandedPath + + ".meaninglessKey"); // We add meaningless Key since calcite json_insert can only + // insert when the path point to null + expands.add(keys.get(i + 1)); + } + } + return JsonFunctions.jsonInsert(jsonStr, expands.toArray()); } public static String jsonAppendIfArray(Object json, List pathValueMap, boolean isExtend) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java index 3907f2c39b4..365a77c21b0 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java @@ -5,9 +5,8 @@ package org.opensearch.sql.expression.function.jsonUDF; -import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; -import java.util.Arrays; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; @@ -15,10 +14,10 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; +import org.apache.calcite.sql.SqlJsonConstructorNullClause; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; /** @@ -37,13 +36,7 @@ public JsonArrayFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return sqlOperatorBinding -> { - RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - return createArrayType( - typeFactory, - typeFactory.createTypeWithNullability(typeFactory.createSqlType(SqlTypeName.ANY), true), - true); - }; + return STRING_FORCE_NULLABLE; } public static class JsonArrayImplementor implements NotNullImplementor { @@ -57,6 +50,6 @@ public Expression implement( } public static Object eval(Object... args) { - return Arrays.asList(args); + return JsonFunctions.jsonArray(SqlJsonConstructorNullClause.NULL_ON_NULL, args); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java index b905831fc2d..1e8dcb8feac 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java @@ -5,14 +5,11 @@ package org.opensearch.sql.expression.function.jsonUDF; +import static org.apache.calcite.runtime.JsonFunctions.jsonRemove; import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.jayway.jsonpath.DocumentContext; -import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.PathNotFoundException; import java.util.Arrays; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -50,24 +47,11 @@ public Expression implement( public static Object eval(Object... args) throws JsonProcessingException { List jsonPaths = Arrays.asList(args).subList(1, args.length); - JsonNode root = verifyInput(args[0]); - DocumentContext ctx; - if (args[0] instanceof String) { - ctx = JsonPath.parse(args[0].toString()); - } else { - ctx = JsonPath.parse(args[0]); - } - for (Object originalPath : jsonPaths) { - String jsonPath = convertToJsonPath(originalPath.toString()); - try { - Object matches = ctx.read(jsonPath); // verify whether it's a valid path - } catch (PathNotFoundException e) { - continue; - } - // Resolve path tokens - PathTokenizer tokenizer = new PathTokenizer(jsonPath); - root = deletePath(root, tokenizer); - } - return root.toString(); + String[] pathSpecs = + jsonPaths.stream() + .map(Object::toString) + .map(JsonUtils::convertToJsonPath) + .toArray(String[]::new); + return jsonRemove(args[0].toString(), pathSpecs); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java index 1c1f6970cb8..b40b34e223e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -6,9 +6,11 @@ package org.opensearch.sql.expression.function.jsonUDF; import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; -import static org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl.jsonAppendIfArray; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -18,6 +20,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -49,8 +52,36 @@ public static Object eval(Object... args) throws JsonProcessingException { List keys = Arrays.asList(args).subList(1, args.length); if (keys.size() % 2 != 0) { throw new RuntimeException( - "Json extend function needs corresponding path and values, but current get: " + keys); + "Json append function needs corresponding path and values, but current get: " + keys); } - return jsonAppendIfArray(jsonStr, keys, true); + JsonNode root = verifyInput(args[0]); + List expands = new ArrayList<>(); + for (int i = 0; i < keys.size(); i += 2) { + List expandedPaths = expandJsonPath(root, convertToJsonPath(keys.get(i).toString())); + for (String expandedPath : expandedPaths) { + Object value = keys.get(i + 1); + if (value instanceof List targetValues) { + for (Object targetValue : targetValues) { + expands.add(expandedPath + ".meaninglessKey"); + expands.add(targetValue); + } + } else if (value instanceof String stringValue) { + try { + List targetValues = gson.fromJson(stringValue, List.class); + for (Object targetValue : targetValues) { + expands.add(expandedPath + ".meaninglessKey"); + expands.add(targetValue); + } + } catch (Exception e) { + expands.add(expandedPath + ".meaninglessKey"); + expands.add(value); + } + } else { + expands.add(expandedPath + ".meaninglessKey"); + expands.add(value); + } + } + } + return JsonFunctions.jsonInsert(jsonStr, expands.toArray()); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index c7949ae9055..8f9daccd935 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -5,26 +5,28 @@ package org.opensearch.sql.expression.function.jsonUDF; -import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.convertToJsonPath; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; +import static org.apache.calcite.sql.SqlJsonQueryEmptyOrErrorBehavior.NULL; +import static org.apache.calcite.sql.SqlJsonQueryWrapperBehavior.WITHOUT_ARRAY; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; -import com.jayway.jsonpath.JsonPath; +import com.fasterxml.jackson.databind.JsonNode; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; +import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.SqlJsonValueEmptyOrErrorBehavior; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; public class JsonExtractFunctionImpl extends ImplementorUDF { @@ -34,17 +36,7 @@ public JsonExtractFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return sqlOperatorBinding -> { - RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - RelDataType varcharType = - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.VARCHAR), true); - if (sqlOperatorBinding.collectOperandTypes().size() > 2) { - return createArrayType(typeFactory, varcharType, true); - } else { - return varcharType; - } - }; + return STRING_FORCE_NULLABLE; } public static class JsonExtractImplementor implements NotNullImplementor { @@ -63,27 +55,58 @@ public static Object eval(Object... args) { if (args.length < 2) { return null; } - Object value = args[0]; + String jsonStr = (String) args[0]; + List jsonPaths = Arrays.asList(args).subList(1, args.length); + /* + JsonNode root = verifyInput(args[0]); + List expands = new ArrayList<>(); + List results = new ArrayList<>(); + for (Object key : keys) { + expands.addAll(expandJsonPath(root, convertToJsonPath(key.toString()))); + } + */ + JsonNode root = verifyInput(args[0]); + List expands = new ArrayList<>(); List results = new ArrayList<>(); - List paths = Arrays.asList(args).subList(1, args.length); - for (Object path : paths) { - String jsonPath = convertToJsonPath(path.toString()); - try { - Object result; - if (value instanceof String) { - result = JsonPath.read((String) value, jsonPath); - } else { - result = JsonPath.read(value, jsonPath); - } - result = result != null ? gson.toJson(result) : null; - results.add(result); - } catch (Exception e) { - results.add(null); - } + for (Object key : jsonPaths) { + expands.addAll(expandJsonPath(root, convertToJsonPath(key.toString()))); + } + List modeExpands = new ArrayList<>(); + for (String expand : expands) { + modeExpands.add(" lax " + expand); + } + JsonFunctions.StatefulFunction a = new JsonFunctions.StatefulFunction(); + for (String pathSpec : modeExpands) { + Object queryResult = a.jsonQuery(jsonStr, pathSpec, WITHOUT_ARRAY, NULL, NULL, false); + Object valueResult = + a.jsonValue( + jsonStr, + pathSpec, + SqlJsonValueEmptyOrErrorBehavior.NULL, + null, + SqlJsonValueEmptyOrErrorBehavior.NULL, + null); + results.add(queryResult != null ? queryResult : valueResult); } - if (paths.size() > 1) { - return results; + if (expands.size() == 1) { + return doJsonize(results.getFirst()); + } + return doJsonize(results); + } + + private static boolean isScalarObject(Object obj) { + if (obj instanceof Collection) { + return false; + } else { + return !(obj instanceof Map); + } + } + + private static String doJsonize(Object candidate) { + if (isScalarObject(candidate)) { + return candidate.toString(); + } else { + return JsonFunctions.jsonize(candidate); } - return results.get(0); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java index 13db90c2bc8..c5095c94201 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java @@ -5,7 +5,6 @@ package org.opensearch.sql.expression.function.jsonUDF; -import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; @@ -14,6 +13,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; @@ -47,13 +47,9 @@ public Expression implement( public static Object eval(Object... args) { assert args.length == 1 : "Json only accept one argument"; - ObjectMapper mapper = new ObjectMapper(); - Object value = args[0]; - try { - mapper.readTree(value.toString()); // try parse as JSON - return value; - } catch (Exception e) { - return null; + if (JsonFunctions.isJsonValue(args[0].toString())) { + return args[0].toString(); } + return null; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java index 7073bfead84..b9ef8225fbf 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java @@ -5,24 +5,19 @@ package org.opensearch.sql.expression.function.jsonUDF; -import static org.apache.calcite.sql.type.SqlTypeUtil.createArrayType; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; -import com.google.gson.JsonSyntaxException; -import java.util.Arrays; import java.util.List; -import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; public class JsonKeysFunctionImpl extends ImplementorUDF { @@ -32,14 +27,7 @@ public JsonKeysFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return sqlOperatorBinding -> { - RelDataTypeFactory typeFactory = sqlOperatorBinding.getTypeFactory(); - return createArrayType( - typeFactory, - typeFactory.createTypeWithNullability( - typeFactory.createSqlType(SqlTypeName.VARCHAR), true), - true); - }; + return STRING_FORCE_NULLABLE; } public static class JsonKeysImplementor implements NotNullImplementor { @@ -56,13 +44,10 @@ public Expression implement( public static Object eval(Object... args) { assert args.length == 1 : "Json keys only accept one argument"; - String value = (String) args[0]; - try { - Map map = gson.fromJson(value, Map.class); - List demo = Arrays.asList(map.keySet().toArray()); - return Arrays.asList(map.keySet().toArray()); - } catch (JsonSyntaxException e) { + String value = JsonFunctions.jsonKeys(args[0].toString()); + if (value.equals("null")) { return null; } + return value; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java index 4dece7431ef..30af2503a16 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java @@ -5,20 +5,20 @@ package org.opensearch.sql.expression.function.jsonUDF; -import java.util.HashMap; +import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; + import java.util.List; -import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; +import org.apache.calcite.sql.SqlJsonConstructorNullClause; import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.calcite.sql.type.SqlTypeName; import org.opensearch.sql.expression.function.ImplementorUDF; public class JsonObjectFunctionImpl extends ImplementorUDF { @@ -28,12 +28,7 @@ public JsonObjectFunctionImpl() { @Override public SqlReturnTypeInference getReturnTypeInference() { - return opBinding -> { - RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); - return typeFactory.createMapType( - typeFactory.createSqlType(SqlTypeName.VARCHAR), - typeFactory.createSqlType(SqlTypeName.ANY)); - }; + return STRING_FORCE_NULLABLE; } public static class JsonObjectImplementor implements NotNullImplementor { @@ -49,10 +44,6 @@ public Expression implement( } public static Object eval(Object... args) { - Map map = new HashMap<>(); - for (int i = 0; i < args.length; i += 2) { - map.put(args[i], args[i + 1]); - } - return map; + return JsonFunctions.jsonObject(SqlJsonConstructorNullClause.NULL_ON_NULL, args); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java index b5d9e6d3d9d..5c9a95d2b0c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -9,13 +9,7 @@ import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.jayway.jsonpath.Configuration; -import com.jayway.jsonpath.DocumentContext; -import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.PathNotFoundException; -import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; -import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -25,6 +19,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -54,63 +49,15 @@ public Expression implement( public static Object eval(Object... args) { String jsonStr = (String) args[0]; List keys = Arrays.asList(args).subList(1, args.length); - if (keys.size() % 2 != 0) { - throw new RuntimeException( - "Json set function needs corresponding path and values, but current get: " + keys); - } - return jsonSetIfParentObject(jsonStr, keys); - } - - public static String jsonSetIfParentObject(Object json, List fullPathToValue) { - try { - JsonNode root = verifyInput(json); - - Configuration conf = - Configuration.builder() - .jsonProvider(new JacksonJsonNodeJsonProvider()) - .mappingProvider(new JacksonMappingProvider()) - .build(); - - DocumentContext context = JsonPath.using(conf).parse(root); - - for (int index = 0; index < fullPathToValue.size(); index += 2) { - String fullPath = convertToJsonPath(fullPathToValue.get(index).toString()); - Object value = fullPathToValue.get(index + 1); - - // Split: "$.a.b.d" -> parentPath = "$.a.b", field = "d" - int lastDot = fullPath.lastIndexOf('.'); - if (lastDot <= 1 || lastDot == fullPath.length() - 1) { - continue; // Invalid path - } - - String parentPath = fullPath.substring(0, lastDot); - String fieldName = fullPath.substring(lastDot + 1); - - JsonNode targets; - try { - targets = context.read(parentPath); - } catch (PathNotFoundException e) { - continue; // parent path not found - } - - if (JsonPath.isPathDefinite(parentPath)) { - if (targets instanceof ObjectNode objectNode) { - objectNode.set(fieldName, objectMapper.valueToTree(value)); - } - } else { - // Some * inside. an arrayNode returned - for (int i = 0; i < targets.size(); i++) { - JsonNode target = targets.get(i); - if (target instanceof ObjectNode objectNode) { - objectNode.set(fieldName, objectMapper.valueToTree(value)); - } - } - } + JsonNode root = verifyInput(args[0]); + List expands = new ArrayList<>(); + for (int i = 0; i < keys.size(); i += 2) { + List expandedPaths = expandJsonPath(root, convertToJsonPath(keys.get(i).toString())); + for (String expandedPath : expandedPaths) { + expands.add(expandedPath); + expands.add(keys.get(i + 1)); } - - return root.toString(); - } catch (Exception e) { - throw new RuntimeException("jsonSetIfParentObject failed", e); } + return JsonFunctions.jsonSet(jsonStr, expands.toArray()); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java index 04f8ba64245..7343baccaa2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java @@ -7,15 +7,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.gson.Gson; import java.util.ArrayList; import java.util.List; public class JsonUtils { static ObjectMapper objectMapper = new ObjectMapper(); - static Gson gson = new Gson(); + public static Gson gson = new Gson(); /** * @param input candidate json path like a.b{}.c{2} @@ -57,99 +55,84 @@ public static String convertToJsonPath(String input) { return sb.toString(); } - static JsonNode deletePath(JsonNode node, PathTokenizer tokenizer) { - if (!tokenizer.hasNext()) return node; - - PathToken token = tokenizer.next(); - - if (token.key.equals("$")) { - return deletePath(node, tokenizer); - } - if (node instanceof ArrayNode arr && token.isIndex) { - if (token.key.equals("*")) { - for (int i = arr.size() - 1; i >= 0; i--) { - deletePath(arr.get(i), tokenizer.cloneFromNext()); - } + public static JsonNode verifyInput(Object input) { + try { + JsonNode root; + if (input instanceof String) { + root = objectMapper.readTree(input.toString()); } else { - int idx = Integer.parseInt(token.key); - if (!tokenizer.hasNext()) { - if (idx >= 0 && idx < arr.size()) arr.remove(idx); - } else { - if (idx >= 0 && idx < arr.size()) { - deletePath(arr.get(idx), tokenizer); - } - } - } - } else if (node instanceof ObjectNode obj && !token.isIndex) { - if (!tokenizer.hasNext()) { - obj.remove(token.key); - } else if (obj.has(token.key)) { - deletePath(obj.get(token.key), tokenizer); + root = objectMapper.valueToTree(input); } + return root; + } catch (Exception e) { + throw new RuntimeException("fail to parse input", e); } - - return node; } - // Tokenizer for JSONPath like $[0].cities[1] - public static class PathTokenizer { - private final List tokens; - private int index = 0; + public static List expandJsonPath(JsonNode root, String rawPath) { + // Remove only leading "$." or "$" + String cleanedPath = rawPath.replaceFirst("^\\$\\.", "").replaceFirst("^\\$", ""); - public PathTokenizer(String path) { - this.tokens = new ArrayList<>(); + String[] parts = cleanedPath.split("\\."); + return expand(root, parts, 0, "$"); + } - // normalize brackets (a[1] => a.[1]) - String normalized = path.replaceAll("\\[", ".["); + private static List expand( + JsonNode currentNode, String[] parts, int index, String prefix) { + if (index >= parts.length || currentNode == null) { + return List.of(prefix); + } - for (String raw : normalized.split("\\.")) { - if (raw.isEmpty()) continue; + String part = parts[index]; + List results = new ArrayList<>(); - if (raw.startsWith("[") && raw.endsWith("]")) { - tokens.add(new PathToken(raw.substring(1, raw.length() - 1), true)); - } else { - tokens.add(new PathToken(raw, false)); + if (part.endsWith("[*]")) { + String field = part.substring(0, part.length() - 3); + JsonNode arrayNode; + if (field.isEmpty()) { + arrayNode = currentNode; + } else { + arrayNode = currentNode.get(field); + } + if (arrayNode != null && arrayNode.isArray()) { + for (int i = 0; i < arrayNode.size(); i++) { + String newPrefix = prefix + "." + field + "[" + i + "]"; + results.addAll(expand(arrayNode.get(i), parts, index + 1, newPrefix)); } } - } - - public boolean hasNext() { - return index < tokens.size(); - } - - public PathToken next() { - return tokens.get(index++); - } - - public PathTokenizer cloneFromNext() { - PathTokenizer clone = new PathTokenizer(""); - clone.tokens.addAll(this.tokens); - clone.index = this.index; - return clone; - } - } - - static class PathToken { - public final String key; - public final boolean isIndex; - - public PathToken(String key, boolean isIndex) { - this.key = key; - this.isIndex = isIndex; - } - } - - public static JsonNode verifyInput(Object input) { - try { - JsonNode root; - if (input instanceof String) { - root = objectMapper.readTree(input.toString()); + } else if (part.endsWith("]")) { + int leftBracketIndex = part.lastIndexOf('['); + String field = part.substring(0, part.length() - 3); + JsonNode arrayNode; + if (field.isEmpty()) { + arrayNode = currentNode; } else { - root = objectMapper.valueToTree(input); + arrayNode = currentNode.get(field); } - return root; - } catch (Exception e) { - throw new RuntimeException("fail to parse input", e); + Boolean arrayFlag = false; + if (leftBracketIndex > -1 + && part.substring(leftBracketIndex + 1, part.length() - 1).matches("-?\\d+")) { + int currentIndex = + Integer.parseInt(part.substring(leftBracketIndex + 1, part.length() - 1)); + if (arrayNode != null && arrayNode.isArray()) { + if (arrayNode.size() > currentIndex) { + String newPrefix = prefix + "." + part; + results.addAll(expand(arrayNode.get(currentIndex), parts, index + 1, newPrefix)); + arrayFlag = true; + } + } + } + if (!arrayFlag) { + JsonNode next = currentNode.get(part); + String newPrefix = prefix + "." + part; + results.addAll(expand(next, parts, index + 1, newPrefix)); + } + } else { // ends with string + JsonNode next = currentNode.get(part); + String newPrefix = prefix + "." + part; + results.addAll(expand(next, parts, index + 1, newPrefix)); } + + return results; } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java index e751b3c34a8..ff141656918 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java @@ -6,9 +6,9 @@ package org.opensearch.sql.expression.function.jsonUDF; import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; -import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; import java.util.List; +import java.util.Map; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexImpTable; @@ -16,6 +16,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.JsonFunctions; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; @@ -43,6 +44,9 @@ public Expression implement( } public static Object eval(Object... args) { - return gson.toJson(args[0]); + if (args[0] instanceof Map) { + return JsonFunctions.jsonize(args[0]); + } + return null; } } diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 7f061ed0da6..e9814d7b431 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -448,8 +448,8 @@ integTest { dependsOn ':opensearch-sql-plugin:bundlePlugin' if(getOSFamilyType() != "windows") { - //dependsOn startPrometheus - //finalizedBy stopPrometheus + dependsOn startPrometheus + finalizedBy stopPrometheus } // enable calcite codegen in IT diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 25cce34d940..0aeeef5335a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -5,14 +5,13 @@ package org.opensearch.sql.calcite.standalone; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.gson; import static org.opensearch.sql.legacy.TestsConstants.*; import static org.opensearch.sql.util.MatcherUtils.*; import static org.opensearch.sql.util.MatcherUtils.rows; -import com.google.gson.Gson; import java.io.IOException; import java.util.List; -import java.util.Map; import org.json.JSONObject; import org.junit.Ignore; import org.junit.jupiter.api.Test; @@ -29,6 +28,7 @@ public void init() throws IOException { loadIndex(Index.PEOPLE2); loadIndex(Index.BANK); loadIndex(Index.JSON_TEST); + loadIndex(Index.GAME_OF_THRONES); } @Test @@ -54,13 +54,9 @@ public void testJsonObject() { + " json_object('inner', 123.45))| fields a, b | head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "struct"), schema("b", "struct")); + verifySchema(actual, schema("a", "string"), schema("b", "string")); - verifyDataRows( - actual, - rows( - new Gson().fromJson("{\"key\":123.45}", Map.class), - new Gson().fromJson("{\"outer\":{\"inner\":123.45}}", Map.class))); + verifyDataRows(actual, rows("{\"key\":123.45}", "{\"outer\":\"{\\\"inner\\\":123.45}\"}")); } @Test @@ -71,9 +67,9 @@ public void testJsonArray() { "source=%s | eval a = json_array(1, 2, 0, -1, 1.1, -0.11)| fields a | head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "array")); + verifySchema(actual, schema("a", "string")); - verifyDataRows(actual, rows(List.of(1.0, 2.0, 0, -1.0, 1.1, -0.11))); + verifyDataRows(actual, rows(gson.toJson(List.of(1, 2, 0, -1, 1.1, -0.11)))); } @Test @@ -85,9 +81,9 @@ public void testJsonArrayWithDifferentType() { + " head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "array")); + verifySchema(actual, schema("a", "string")); - verifyDataRows(actual, rows(List.of(1, "123", Map.of("name", 3)))); + verifyDataRows(actual, rows("[1,\"123\",\"{\\\"name\\\":3}\"]")); } @Test @@ -95,13 +91,13 @@ public void testToJsonString() { JSONObject actual = executeQuery( String.format( - "source=%s | eval a = to_json_string(json_array(1, 2, 0, -1, 1.1, -0.11))| fields a" - + " | head 1", - TEST_INDEX_PEOPLE2)); + "source=%s | eval a = to_json_string(name) | fields a" + " | head 1", + TEST_INDEX_GAME_OF_THRONES)); verifySchema(actual, schema("a", "string")); - verifyDataRows(actual, rows("[1,2,0,-1,1.1,-0.11]")); + verifyDataRows( + actual, rows("{\"firstname\":\"Daenerys\",\"lastname\":\"Targaryen\",\"ofHerName\":1}")); } @Test @@ -161,10 +157,10 @@ public void testJsonExtractWithNewPath() { + "]\n" + "},\n" + "{\n" - + "\"name\": \"San Francisco\",\n" + + "\"name\":\"San Francisco\",\n" + "\"Bridges\":[\n" + "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0},\n" - + "{\"name\":\"Bay Bridge\", \"length\":23556.0}\n" + + "{\"name\":\"Bay Bridge\",\"length\":23556.0}\n" + "]\n" + "}\n" + "]"; @@ -181,7 +177,7 @@ public void testJsonExtractWithNewPath() { verifyDataRows( actual, rows( - new Gson().toJson(new Gson().fromJson(candidate, List.class)), + gson.toJson(gson.fromJson(candidate, List.class)), "8981.0", "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0}")); } @@ -196,9 +192,9 @@ public void testJsonKeys() { + " =json_keys('[1,2,3,{\"f1\":1,\"f2\":[5,6]},4]') | fields a,b | head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "array"), schema("b", "array")); + verifySchema(actual, schema("a", "string"), schema("b", "string")); - verifyDataRows(actual, rows(List.of("f1", "f2"), null)); + verifyDataRows(actual, rows("[\"f1\",\"f2\"]", null)); } @Test @@ -223,9 +219,9 @@ public void testArray() { "source=%s | eval a =array(1, 2, 0, -1, 1.1, -0.11) | fields a | head 1", TEST_INDEX_PEOPLE2)); - verifySchema(actual, schema("a", "array")); + verifySchema(actual, schema("a", "string")); - verifyDataRows(actual, rows(List.of(1.0, 2.0, 0, -1.0, 1.1, -0.11))); + verifyDataRows(actual, rows("[1,2,0,-1,1.1,-0.11]")); } @Test @@ -321,9 +317,9 @@ public void testJsonAppend() { verifyDataRows( actual, rows( - "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tomy\",\"rank\":5}]}", + "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},\"{\\\"name\\\":\\\"Tomy\\\",\\\"rank\\\":5}\"]}", "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}", - "{\"school\":{\"teacher\":[\"Alice\",[\"Tom\",\"Walt\"]],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}")); + "{\"school\":{\"teacher\":[\"Alice\",\"[\\\"Tom\\\",\\\"Walt\\\"]\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}")); } @Test @@ -345,19 +341,8 @@ public void testJsonExtend() { verifyDataRows( actual, rows( - "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},{\"name\":\"Tommy\",\"rank\":5}]}", + "{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2},\"{\\\"name\\\":\\\"Tommy\\\",\\\"rank\\\":5}\"]}", "{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}", "{\"school\":{\"teacher\":[\"Alice\",\"Tom\",\"Walt\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}")); } - - @Test - public void test_cast_json() throws IOException { - JSONObject result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | eval casted=cast(json_string as json)" - + " | fields test_name", - TEST_INDEX_JSON_TEST)); - assertEquals(1, 1); - } } From 0e43eb940f2001619102140438d5adee76a0934a Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 16 May 2025 13:40:28 +0800 Subject: [PATCH 22/50] remove useless change Signed-off-by: xinyual --- .../function/PPLBuiltinOperators.java | 9 --- .../expression/function/PPLFuncImpTable.java | 5 -- .../collectionUDF/ForallFunctionImpl.java | 58 ------------------- .../function/collectionUDF/LambdaUtils.java | 40 ------------- .../jsonUDF/JsonAppendFunctionImpl.java | 4 +- .../jsonUDF/JsonArrayFunctionImpl.java | 55 ------------------ .../jsonUDF/JsonExtendFunctionImpl.java | 2 +- .../jsonUDF/JsonExtractFunctionImpl.java | 2 +- .../jsonUDF/JsonObjectFunctionImpl.java | 49 ---------------- .../function/jsonUDF/JsonSetFunctionImpl.java | 2 +- .../function/jsonUDF/JsonUtils.java | 18 ++++-- .../jsonUDF/JsonValidFunctionImpl.java | 53 ----------------- 12 files changed, 17 insertions(+), 280 deletions(-) delete mode 100644 core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java delete mode 100644 core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/LambdaUtils.java delete mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java delete mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java delete mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 8f261f1a64a..787acc5b4a7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -26,18 +26,14 @@ import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.expression.datetime.DateTimeFunctions; -import org.opensearch.sql.expression.function.collectionUDF.ForallFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonAppendFunctionImpl; -import org.opensearch.sql.expression.function.jsonUDF.JsonArrayFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonArrayLengthFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonDeleteFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonExtendFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonExtractFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl; -import org.opensearch.sql.expression.function.jsonUDF.JsonObjectFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl; -import org.opensearch.sql.expression.function.jsonUDF.JsonValidFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.ToJsonStringFunctionImpl; import org.opensearch.sql.expression.function.udf.CryptographicFunction; import org.opensearch.sql.expression.function.udf.datetime.AddSubDateFunction; @@ -72,8 +68,6 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { // Math functions public static final SqlOperator SPAN = new SpanFunctionImpl().toUDF("SPAN"); public static final SqlOperator JSON = new JsonFunctionImpl().toUDF("JSON"); - public static final SqlOperator JSON_OBJECT = new JsonObjectFunctionImpl().toUDF("JSON_OBJECT"); - public static final SqlOperator JSON_ARRAY = new JsonArrayFunctionImpl().toUDF("JSON_ARRAY"); public static final SqlOperator TO_JSON_STRING = new ToJsonStringFunctionImpl().toUDF("TO_JSON_STRING"); public static final SqlOperator JSON_ARRAY_LENGTH = @@ -81,14 +75,11 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlOperator JSON_EXTRACT = new JsonExtractFunctionImpl().toUDF("JSON_EXTRACT"); public static final SqlOperator JSON_KEYS = new JsonKeysFunctionImpl().toUDF("JSON_KEYS"); - public static final SqlOperator JSON_VALID = new JsonValidFunctionImpl().toUDF("JSON_VALID"); - public static final SqlOperator ARRAY = new JsonArrayFunctionImpl().toUDF("ARRAY"); public static final SqlOperator JSON_SET = new JsonSetFunctionImpl().toUDF("JSON_SET"); public static final SqlOperator JSON_DELETE = new JsonDeleteFunctionImpl().toUDF("JSON_DELETE"); public static final SqlOperator JSON_APPEND = new JsonAppendFunctionImpl().toUDF("JSON_APPEND"); public static final SqlOperator JSON_EXTEND = new JsonExtendFunctionImpl().toUDF("JSON_EXTEND"); - public static final SqlOperator FORALL = new ForallFunctionImpl().toUDF("FORALL"); public static final SqlOperator E = new EulerFunction().toUDF("E"); public static final SqlOperator CONV = new ConvFunction().toUDF("CONVERT"); public static final SqlOperator MOD = new ModFunction().toUDF("MOD"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 9e9e6372d1e..cdcb8ba7cc0 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -320,9 +320,6 @@ void populate() { registerOperator(WEEK_OF_YEAR, PPLBuiltinOperators.WEEK); registerOperator(WEEKOFYEAR, PPLBuiltinOperators.WEEK); - // Register Array Function - registerOperator(ARRAY, PPLBuiltinOperators.ARRAY); - // Register Json function register( JSON_ARRAY, @@ -349,8 +346,6 @@ void populate() { registerOperator(JSON_APPEND, PPLBuiltinOperators.JSON_APPEND); registerOperator(JSON_EXTEND, PPLBuiltinOperators.JSON_EXTEND); - registerOperator(ARRAY_FORALL, PPLBuiltinOperators.FORALL); - // Register implementation. // Note, make the implementation an individual class if too complex. register( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java deleted file mode 100644 index e2da4acee61..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/ForallFunctionImpl.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function.collectionUDF; - -import java.util.List; -import org.apache.calcite.adapter.enumerable.NotNullImplementor; -import org.apache.calcite.adapter.enumerable.NullPolicy; -import org.apache.calcite.adapter.enumerable.RexImpTable; -import org.apache.calcite.adapter.enumerable.RexToLixTranslator; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.schema.impl.ScalarFunctionImpl; -import org.apache.calcite.sql.type.ReturnTypes; -import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.opensearch.sql.expression.function.ImplementorUDF; - -public class ForallFunctionImpl extends ImplementorUDF { - public ForallFunctionImpl() { - super(new ForallImplementor(), NullPolicy.ANY); - } - - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return ReturnTypes.BOOLEAN; - } - - public static class ForallImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(ForallFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } - } - - public static Object eval(Object... args) { - org.apache.calcite.linq4j.function.Predicate1 lambdaFunction = - (org.apache.calcite.linq4j.function.Predicate1) args[1]; - List target = (List) args[0]; - try { - for (Object candidate : target) { - if (!(Boolean) lambdaFunction.apply(candidate)) { - return false; - } - } - } catch (Exception e) { - throw new RuntimeException(e); - } - return true; - } -} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/LambdaUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/LambdaUtils.java deleted file mode 100644 index 38aedce6278..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/function/collectionUDF/LambdaUtils.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function.collectionUDF; - -import com.googlecode.aviator.AviatorEvaluator; -import com.googlecode.aviator.Expression; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class LambdaUtils { - public static class SimpleLambda { - private Expression expression; - private List inputVariables; - - public SimpleLambda(String lambda) { - String[] lambdaParts = lambda.split("->"); - if (lambdaParts.length != 2) { - throw new IllegalArgumentException("Invalid lambda expression format"); - } - - String parameter = lambdaParts[0].trim(); - expression = AviatorEvaluator.compile(lambdaParts[1].trim()); - inputVariables = - Arrays.stream(parameter.split(",")).map(String::trim).collect(Collectors.toList()); - } - - public List parameters() { - return inputVariables; - } - - public Object execute(Map variableMap) { - return expression.execute(variableMap); - } - } -} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java index 4122d12bffe..38aa5ebbfcb 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -61,7 +61,7 @@ public static Object eval(Object... args) throws JsonProcessingException { throw new RuntimeException( "Json append function needs corresponding path and values, but current get: " + keys); } - JsonNode root = verifyInput(args[0]); + JsonNode root = convertInputToJsonNode(args[0]); List expands = new ArrayList<>(); for (int i = 0; i < keys.size(); i += 2) { List expandedPaths = expandJsonPath(root, convertToJsonPath(keys.get(i).toString())); @@ -78,7 +78,7 @@ public static Object eval(Object... args) throws JsonProcessingException { public static String jsonAppendIfArray(Object json, List pathValueMap, boolean isExtend) { try { - JsonNode tree = verifyInput(json); + JsonNode tree = convertInputToJsonNode(json); Configuration conf = Configuration.builder() diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java deleted file mode 100644 index 365a77c21b0..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayFunctionImpl.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function.jsonUDF; - -import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; - -import java.util.List; -import org.apache.calcite.adapter.enumerable.NotNullImplementor; -import org.apache.calcite.adapter.enumerable.NullPolicy; -import org.apache.calcite.adapter.enumerable.RexToLixTranslator; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Expressions; -import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.runtime.JsonFunctions; -import org.apache.calcite.sql.SqlJsonConstructorNullClause; -import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.opensearch.sql.expression.function.ImplementorUDF; - -/** - * json_array(...) Creates a JSON ARRAY using a list of values. - * - *

Argument type: - * - *

A can be any kind of value such as string, number, or boolean. Return type: ARRAY - * (Spark ArrayType) It will also do implicit convert when we can find common types. E.g. - * json_array(1, 2, 0, -1, 1.1, -0.11) = [1.0,2.0,0.0,-1.0,1.1,-0.11] - */ -public class JsonArrayFunctionImpl extends ImplementorUDF { - public JsonArrayFunctionImpl() { - super(new JsonArrayImplementor(), NullPolicy.ANY); - } - - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return STRING_FORCE_NULLABLE; - } - - public static class JsonArrayImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - return Expressions.call( - Types.lookupMethod(JsonArrayFunctionImpl.class, "eval", Object[].class), - translatedOperands); - } - } - - public static Object eval(Object... args) { - return JsonFunctions.jsonArray(SqlJsonConstructorNullClause.NULL_ON_NULL, args); - } -} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java index b40b34e223e..e6b97edbd57 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -54,7 +54,7 @@ public static Object eval(Object... args) throws JsonProcessingException { throw new RuntimeException( "Json append function needs corresponding path and values, but current get: " + keys); } - JsonNode root = verifyInput(args[0]); + JsonNode root = convertInputToJsonNode(args[0]); List expands = new ArrayList<>(); for (int i = 0; i < keys.size(); i += 2) { List expandedPaths = expandJsonPath(root, convertToJsonPath(keys.get(i).toString())); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index 8f9daccd935..a5e98d74e58 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -65,7 +65,7 @@ public static Object eval(Object... args) { expands.addAll(expandJsonPath(root, convertToJsonPath(key.toString()))); } */ - JsonNode root = verifyInput(args[0]); + JsonNode root = convertInputToJsonNode(args[0]); List expands = new ArrayList<>(); List results = new ArrayList<>(); for (Object key : jsonPaths) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java deleted file mode 100644 index 30af2503a16..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonObjectFunctionImpl.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function.jsonUDF; - -import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; - -import java.util.List; -import org.apache.calcite.adapter.enumerable.NotNullImplementor; -import org.apache.calcite.adapter.enumerable.NullPolicy; -import org.apache.calcite.adapter.enumerable.RexImpTable; -import org.apache.calcite.adapter.enumerable.RexToLixTranslator; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.runtime.JsonFunctions; -import org.apache.calcite.schema.impl.ScalarFunctionImpl; -import org.apache.calcite.sql.SqlJsonConstructorNullClause; -import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.opensearch.sql.expression.function.ImplementorUDF; - -public class JsonObjectFunctionImpl extends ImplementorUDF { - public JsonObjectFunctionImpl() { - super(new JsonObjectImplementor(), NullPolicy.ALL); - } - - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return STRING_FORCE_NULLABLE; - } - - public static class JsonObjectImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(JsonObjectFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } - } - - public static Object eval(Object... args) { - return JsonFunctions.jsonObject(SqlJsonConstructorNullClause.NULL_ON_NULL, args); - } -} diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java index 5c9a95d2b0c..4c282198177 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -49,7 +49,7 @@ public Expression implement( public static Object eval(Object... args) { String jsonStr = (String) args[0]; List keys = Arrays.asList(args).subList(1, args.length); - JsonNode root = verifyInput(args[0]); + JsonNode root = convertInputToJsonNode(args[0]); List expands = new ArrayList<>(); for (int i = 0; i < keys.size(); i += 2) { List expandedPaths = expandJsonPath(root, convertToJsonPath(keys.get(i).toString())); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java index 7343baccaa2..241178acfda 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java @@ -28,7 +28,6 @@ public static String convertToJsonPath(String input) { char c = input.charAt(i); if (c == '{') { - // 处理 {...} 为数组访问或通配符 int end = input.indexOf('}', i); if (end == -1) throw new IllegalArgumentException("Unmatched { in input"); @@ -43,7 +42,6 @@ public static String convertToJsonPath(String input) { sb.append("."); i++; } else { - // 读取字段名 int start = i; while (i < input.length() && input.charAt(i) != '.' && input.charAt(i) != '{') { i++; @@ -55,7 +53,7 @@ public static String convertToJsonPath(String input) { return sb.toString(); } - public static JsonNode verifyInput(Object input) { + public static JsonNode convertInputToJsonNode(Object input) { try { JsonNode root; if (input instanceof String) { @@ -69,6 +67,14 @@ public static JsonNode verifyInput(Object input) { } } + /** + * The function will expand the json path to eliminate the wildcard *. For example, a[*] would be + * a[0], a[1]... + * + * @param root The json node + * @param rawPath original path + * @return List of expanded paths + */ public static List expandJsonPath(JsonNode root, String rawPath) { // Remove only leading "$." or "$" String cleanedPath = rawPath.replaceFirst("^\\$\\.", "").replaceFirst("^\\$", ""); @@ -86,7 +92,7 @@ private static List expand( String part = parts[index]; List results = new ArrayList<>(); - if (part.endsWith("[*]")) { + if (part.endsWith("[*]")) { // Contains wildcard symbol String field = part.substring(0, part.length() - 3); JsonNode arrayNode; if (field.isEmpty()) { @@ -100,7 +106,7 @@ private static List expand( results.addAll(expand(arrayNode.get(i), parts, index + 1, newPrefix)); } } - } else if (part.endsWith("]")) { + } else if (part.endsWith("]")) { // Normal index symbol int leftBracketIndex = part.lastIndexOf('['); String field = part.substring(0, part.length() - 3); JsonNode arrayNode; @@ -122,7 +128,7 @@ private static List expand( } } } - if (!arrayFlag) { + if (!arrayFlag) { // normal keys JsonNode next = currentNode.get(part); String newPrefix = prefix + "." + part; results.addAll(expand(next, parts, index + 1, newPrefix)); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java deleted file mode 100644 index 759333da854..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonValidFunctionImpl.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function.jsonUDF; - -import com.google.gson.JsonParser; -import com.google.gson.JsonSyntaxException; -import java.util.List; -import org.apache.calcite.adapter.enumerable.NotNullImplementor; -import org.apache.calcite.adapter.enumerable.NullPolicy; -import org.apache.calcite.adapter.enumerable.RexImpTable; -import org.apache.calcite.adapter.enumerable.RexToLixTranslator; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.schema.impl.ScalarFunctionImpl; -import org.apache.calcite.sql.type.ReturnTypes; -import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.opensearch.sql.expression.function.ImplementorUDF; - -public class JsonValidFunctionImpl extends ImplementorUDF { - public JsonValidFunctionImpl() { - super(new JsonValidImplementor(), NullPolicy.ANY); - } - - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return ReturnTypes.BOOLEAN; - } - - public static class JsonValidImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(JsonValidFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } - } - - public static Object eval(Object... args) { - try { - JsonParser.parseString(args[0].toString()); - return true; - } catch (JsonSyntaxException e) { - return false; - } - } -} From d7c1a9a1cb2c55e8e4220fb1b8d377b3dffb69ad Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 16 May 2025 13:43:08 +0800 Subject: [PATCH 23/50] remove useless change Signed-off-by: xinyual --- .../sql/ast/AbstractNodeVisitor.java | 5 ---- .../sql/calcite/CalcitePlanContext.java | 9 -------- .../sql/calcite/CalciteRexNodeVisitor.java | 23 ------------------- 3 files changed, 37 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index eda028a9ea9..39401db606f 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -22,7 +22,6 @@ import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; import org.opensearch.sql.ast.expression.Interval; -import org.opensearch.sql.ast.expression.LambdaFunction; import org.opensearch.sql.ast.expression.Let; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Map; @@ -229,10 +228,6 @@ public T visitLet(Let node, C context) { return visitChildren(node, context); } - public T visitLambdaFunction(LambdaFunction node, C context) { - return visitChildren(node, context); - } - public T visitSort(Sort node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index e2e1a3e5242..8d6f09b83f0 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -8,16 +8,13 @@ import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; import java.sql.Connection; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Stack; import java.util.function.BiFunction; import lombok.Getter; import lombok.Setter; import org.apache.calcite.rex.RexCorrelVariable; -import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.tools.FrameworkConfig; import org.apache.calcite.tools.RelBuilder; @@ -34,7 +31,6 @@ public class CalcitePlanContext { public final ExtendedRexBuilder rexBuilder; public final FunctionProperties functionProperties; public final QueryType queryType; - @Getter public Map temparolInputmap; @Getter @Setter private boolean isResolvingJoinCondition = false; @Getter @Setter private boolean isResolvingSubquery = false; @@ -57,11 +53,6 @@ private CalcitePlanContext(FrameworkConfig config, QueryType queryType) { this.relBuilder = CalciteToolsHelper.create(config, TYPE_FACTORY, connection); this.rexBuilder = new ExtendedRexBuilder(relBuilder.getRexBuilder()); this.functionProperties = new FunctionProperties(QueryType.PPL); - this.temparolInputmap = new HashMap<>(); - } - - public void putTemparolInputmap(String name, RexLambdaRef input) { - this.temparolInputmap.put(name, input); } public RexNode resolveJoinCondition( diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 4bc707f3337..3b494b91826 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -41,7 +41,6 @@ import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.In; import org.opensearch.sql.ast.expression.Interval; -import org.opensearch.sql.ast.expression.LambdaFunction; import org.opensearch.sql.ast.expression.Let; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Not; @@ -340,28 +339,6 @@ public RexNode visitLet(Let node, CalcitePlanContext context) { return context.relBuilder.alias(expr, node.getVar().getField().toString()); } - @Override - public RexNode visitLambdaFunction(LambdaFunction node, CalcitePlanContext context) { - List names = node.getFuncArgs(); - List args = new ArrayList<>(); - for (int i = 0; i < names.size(); i++) { - context.putTemparolInputmap( - names.get(i).toString(), - new RexLambdaRef( - i, - names.get(i).toString(), - context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY))); - args.add( - new RexLambdaRef( - i, - names.get(i).toString(), - context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.ANY))); - } - RexNode body = node.getFunction().accept(this, context); - RexNode lambdaNode = context.rexBuilder.makeLambdaCall(body, args); - return lambdaNode; - } - @Override public RexNode visitFunction(Function node, CalcitePlanContext context) { List arguments = From 330a989b4908f6a2114dbe67dc4e4d58584f3e38 Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 16 May 2025 13:46:56 +0800 Subject: [PATCH 24/50] remove useless change Signed-off-by: xinyual --- .../utils/UserDefinedFunctionUtils.java | 24 ------------------- .../expression/function/PPLFuncImpTable.java | 17 ------------- integ-test/build.gradle | 2 +- .../CalcitePPLJsonBuiltinFunctionIT.java | 15 +----------- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 3 --- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 5 ---- 6 files changed, 2 insertions(+), 64 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index 5891f72576f..f5e2c0080ef 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -21,27 +21,20 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; -import org.apache.calcite.linq4j.tree.Types; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; -import org.apache.calcite.schema.ScalarFunction; import org.apache.calcite.schema.impl.AggregateFunctionImpl; -import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.SqlIdentifier; import org.apache.calcite.sql.SqlKind; -import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; -import org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.util.Optionality; import org.opensearch.sql.calcite.udf.UserDefinedAggFunction; -import org.opensearch.sql.calcite.udf.UserDefinedFunction; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.executor.QueryType; @@ -80,23 +73,6 @@ public static RelBuilder.AggCall TransferUserDefinedAggFunction( return relBuilder.aggregateCall(sqlUDAF, addArgList); } - public static SqlOperator TransferUserDefinedFunction( - Class UDF, - String functionName, - SqlReturnTypeInference returnType) { - final ScalarFunction udfFunction = - ScalarFunctionImpl.create(Types.lookupMethod(UDF, "eval", Object[].class)); - SqlIdentifier udfLtrimIdentifier = - new SqlIdentifier(Collections.singletonList(functionName), null, SqlParserPos.ZERO, null); - return new SqlUserDefinedFunction( - udfLtrimIdentifier, - SqlKind.OTHER_FUNCTION, - returnType, - InferTypes.ANY_NULLABLE, - null, - udfFunction); - } - static SqlReturnTypeInference getReturnTypeInferenceForArray() { return opBinding -> { RelDataTypeFactory typeFactory = opBinding.getTypeFactory(); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index cdcb8ba7cc0..67d51be24db 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -84,23 +84,6 @@ default List getParams() { } } - public interface FunctionImpAny extends FunctionImp { - RexNode resolve(RexBuilder builder, List additonalArgs, RexNode... args); - - @Override - default RexNode resolve(RexBuilder builder, RexNode... args) { - if (args.length != 2) { - throw new IllegalArgumentException("This function requires exactly 2 arguments"); - } - return resolve(builder, args[0], args[1]); - } - - @Override - default List getParams() { - return null; - } - } - /** The singleton instance. */ public static final PPLFuncImpTable INSTANCE; diff --git a/integ-test/build.gradle b/integ-test/build.gradle index e9814d7b431..7b5bade79ce 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -453,7 +453,7 @@ integTest { } // enable calcite codegen in IT - systemProperty 'calcite.debug', 'true' + systemProperty 'calcite.debug', 'false' systemProperty 'org.codehaus.janino.source_debugging.enable', 'false' systemProperty 'org.codehaus.janino.source_debugging.dir', calciteCodegen diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 0aeeef5335a..0d2c1f11ab7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -211,19 +211,6 @@ public void testJsonValid() { verifyDataRows(actual, rows(true, false)); } - @Test - public void testArray() { - JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a =array(1, 2, 0, -1, 1.1, -0.11) | fields a | head 1", - TEST_INDEX_PEOPLE2)); - - verifySchema(actual, schema("a", "string")); - - verifyDataRows(actual, rows("[1,2,0,-1,1.1,-0.11]")); - } - @Test public void testJsonSet() { JSONObject actual = @@ -333,7 +320,7 @@ public void testJsonExtend() { + " json_extend('{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}'," + " 'teacher', 'Tom', 'teacher', 'Walt'),c =" + " json_extend('{\"school\":{\"teacher\":[\"Alice\"],\"student\":[{\"name\":\"Bob\",\"rank\":1},{\"name\":\"Charlie\",\"rank\":2}]}}'," - + " 'school.teacher', array(\"Tom\", \"Walt\"))| fields a, b, c | head 1", + + " 'school.teacher', json_array(\"Tom\", \"Walt\"))| fields a, b, c | head 1", TEST_INDEX_PEOPLE2)); verifySchema(actual, schema("a", "string"), schema("b", "string"), schema("c", "string")); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 88536d4f2eb..164e91c2140 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -367,9 +367,6 @@ ISNOTNULL: 'ISNOTNULL'; CIDRMATCH: 'CIDRMATCH'; BETWEEN: 'BETWEEN'; -// COLLECTION FUNCTIONS -ARRAY: 'ARRAY'; - // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 2a90c4b89ce..2dcffc4c5b4 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -580,7 +580,6 @@ evalFunctionName | cryptographicFunctionName | jsonFunctionName | geoipFunctionName - | collectionFunctionName ; functionArgs @@ -699,10 +698,6 @@ trigonometricFunctionName | TAN ; -collectionFunctionName - : ARRAY - ; - jsonFunctionName : JSON | JSON_OBJECT From ca49008e5b4a4f6fedd959ff8efae90fb2317713 Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 16 May 2025 13:48:56 +0800 Subject: [PATCH 25/50] remove useless change Signed-off-by: xinyual --- .../org/opensearch/sql/calcite/CalciteRexNodeVisitor.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 3b494b91826..f9f04ee88d7 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -277,10 +277,6 @@ public RexNode visitQualifiedName(QualifiedName node, CalcitePlanContext context // 2. resolve QualifiedName in non-join condition String qualifiedName = node.toString(); List currentFields = context.relBuilder.peek().getRowType().getFieldNames(); - Map map = context.getTemparolInputmap(); - if (map.containsKey(qualifiedName)) { - return map.get(qualifiedName); - } if (currentFields.contains(qualifiedName)) { // 2.1 resolve QualifiedName from stack top return context.relBuilder.field(qualifiedName); From 0a648bf2da8dc07ee28b1fd051b71f9848b7f382 Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 16 May 2025 13:52:15 +0800 Subject: [PATCH 26/50] remove useless change Signed-off-by: xinyual --- .../sql/ast/expression/LambdaFunction.java | 47 ------------------- .../sql/calcite/CalciteRexNodeVisitor.java | 2 - integ-test/build.gradle | 4 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 - 4 files changed, 2 insertions(+), 52 deletions(-) delete mode 100644 core/src/main/java/org/opensearch/sql/ast/expression/LambdaFunction.java diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/LambdaFunction.java b/core/src/main/java/org/opensearch/sql/ast/expression/LambdaFunction.java deleted file mode 100644 index e7eab427652..00000000000 --- a/core/src/main/java/org/opensearch/sql/ast/expression/LambdaFunction.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.ast.expression; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.opensearch.sql.ast.AbstractNodeVisitor; - -/** - * Expression node of lambda function. Params include function name (@funcName) and function - * arguments (@funcArgs) - */ -@Getter -@EqualsAndHashCode(callSuper = false) -@RequiredArgsConstructor -public class LambdaFunction extends UnresolvedExpression { - private final UnresolvedExpression function; - private final List funcArgs; - - @Override - public List getChild() { - List children = new ArrayList<>(); - children.add(function); - children.addAll(funcArgs); - return children; - } - - @Override - public R accept(AbstractNodeVisitor nodeVisitor, C context) { - return nodeVisitor.visitLambdaFunction(this, context); - } - - @Override - public String toString() { - return String.format( - "(%s) -> %s", - funcArgs.stream().map(Object::toString).collect(Collectors.joining(", ")), - function.toString()); - } -} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index f9f04ee88d7..2225a9f8312 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -14,14 +14,12 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import lombok.RequiredArgsConstructor; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.sql.fun.SqlStdOperatorTable; diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 7b5bade79ce..f164db33f3a 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -448,8 +448,8 @@ integTest { dependsOn ':opensearch-sql-plugin:bundlePlugin' if(getOSFamilyType() != "windows") { - dependsOn startPrometheus - finalizedBy stopPrometheus + //dependsOn startPrometheus + //finalizedBy stopPrometheus } // enable calcite codegen in IT diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 2dcffc4c5b4..661ca6c5b6c 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -1043,7 +1043,6 @@ keywordsCanBeId | SPAN | evalFunctionName | jsonFunctionName - | collectionFunctionName | relevanceArgName | intervalUnit | trendlineType From 019b67a5878ab44f9f1c0e9f2ae10845adf3aca3 Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 16 May 2025 15:37:47 +0800 Subject: [PATCH 27/50] change extract implementation Signed-off-by: xinyual --- .../jsonUDF/JsonExtractFunctionImpl.java | 42 ++++------ .../CalcitePPLJsonBuiltinFunctionIT.java | 80 ++++++++++++------- 2 files changed, 68 insertions(+), 54 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index a5e98d74e58..d55f6f52a1c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -55,40 +55,28 @@ public static Object eval(Object... args) { if (args.length < 2) { return null; } + JsonFunctions.StatefulFunction a = new JsonFunctions.StatefulFunction(); String jsonStr = (String) args[0]; List jsonPaths = Arrays.asList(args).subList(1, args.length); - /* - JsonNode root = verifyInput(args[0]); - List expands = new ArrayList<>(); - List results = new ArrayList<>(); - for (Object key : keys) { - expands.addAll(expandJsonPath(root, convertToJsonPath(key.toString()))); - } - */ - JsonNode root = convertInputToJsonNode(args[0]); - List expands = new ArrayList<>(); + List pathSpecs = + jsonPaths.stream() + .map(Object::toString) + .map(JsonUtils::convertToJsonPath) + .map(s -> " lax " + s).toList(); List results = new ArrayList<>(); - for (Object key : jsonPaths) { - expands.addAll(expandJsonPath(root, convertToJsonPath(key.toString()))); - } - List modeExpands = new ArrayList<>(); - for (String expand : expands) { - modeExpands.add(" lax " + expand); - } - JsonFunctions.StatefulFunction a = new JsonFunctions.StatefulFunction(); - for (String pathSpec : modeExpands) { + for (String pathSpec : pathSpecs) { Object queryResult = a.jsonQuery(jsonStr, pathSpec, WITHOUT_ARRAY, NULL, NULL, false); Object valueResult = - a.jsonValue( - jsonStr, - pathSpec, - SqlJsonValueEmptyOrErrorBehavior.NULL, - null, - SqlJsonValueEmptyOrErrorBehavior.NULL, - null); + a.jsonValue( + jsonStr, + pathSpec, + SqlJsonValueEmptyOrErrorBehavior.NULL, + null, + SqlJsonValueEmptyOrErrorBehavior.NULL, + null); results.add(queryResult != null ? queryResult : valueResult); } - if (expands.size() == 1) { + if (jsonPaths.size() == 1) { return doJsonize(results.getFirst()); } return doJsonize(results); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 0d2c1f11ab7..617a5d1d0c0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -115,30 +115,8 @@ public void testJsonArrayLength() { verifyDataRows(actual, rows(4, 5, null)); } - @Ignore @Test public void testJsonExtract() { - JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a = json_extract('{\"a\":\"b\"}', '$.a'), b =" - + " json_extract('{\"a\":[{\"b\":1},{\"b\":2}]}', '$.a[1].b'), c =" - + " json_extract('{\"a\":[{\"b\":1},{\"b\":2}]}', '$.a[*].b'), d =" - + " json_extract('{\"invalid\": \"json\"') | fields a,b,c,d | head 1", - TEST_INDEX_PEOPLE2)); - - verifySchema( - actual, - schema("a", "string"), - schema("b", "string"), - schema("c", "string"), - schema("d", "string")); - - verifyDataRows(actual, rows("b", "2", "[1,2]", null)); - } - - @Test - public void testJsonExtractWithNewPath() { String candidate = "[\n" + "{\n" @@ -152,8 +130,8 @@ public void testJsonExtractWithNewPath() { + "\"name\":\"Venice\",\n" + "\"Bridges\":[\n" + "{\"name\":\"Rialto Bridge\",\"length\":157.0},\n" - + "{\"name\":\"Bridge of Sighs\",\"length\":36.0},\n" - + "{\"name\":\"Ponte della Paglia\"}\n" + + "{\"type\":\"Bridge of Sighs\",\"length\":36.0},\n" + + "{\"type\":\"Ponte della Paglia\"}\n" + "]\n" + "},\n" + "{\n" @@ -168,20 +146,68 @@ public void testJsonExtractWithNewPath() { executeQuery( String.format( "source=%s | head 1 | eval a = json_extract('%s', '{}'), b= json_extract('%s'," - + " '{2}.Bridges{0}.length'), d=json_extract('%s', '{2}.Bridges{0}')| fields a," - + " b, d | head 1", + + " '{2}.Bridges{0}.length'), c=json_extract('%s', '{}.Bridges{}.type'), d=json_extract('%s', '{2}.Bridges{0}')| fields a," + + " b,c, d | head 1", TEST_INDEX_PEOPLE2, candidate, candidate, candidate, candidate)); - verifySchema(actual, schema("a", "string"), schema("b", "string"), schema("d", "string")); + verifySchema(actual, schema("a", "string"), + schema("b", "string"), + schema("c", "string"), + schema("d", "string")); verifyDataRows( actual, rows( gson.toJson(gson.fromJson(candidate, List.class)), "8981.0", + "[\"Bridge of Sighs\",\"Ponte della Paglia\"]", "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0}")); } + @Test + public void testJsonExtractWithMultiplyResult() { + String candidate = + "[\n" + + "{\n" + + "\"name\":\"London\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Tower Bridge\",\"length\":801.0},\n" + + "{\"name\":\"Millennium Bridge\",\"length\":1066.0}\n" + + "]\n" + + "},\n" + + "{\n" + + "\"name\":\"Venice\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Rialto Bridge\",\"length\":157.0},\n" + + "{\"type\":\"Bridge of Sighs\",\"length\":36.0},\n" + + "{\"type\":\"Ponte della Paglia\"}\n" + + "]\n" + + "},\n" + + "{\n" + + "\"name\":\"San Francisco\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0},\n" + + "{\"name\":\"Bay Bridge\",\"length\":23556.0}\n" + + "]\n" + + "}\n" + + "]"; + JSONObject actual = + executeQuery( + String.format( + "source=%s | head 1 | eval c=json_extract('%s', '{}.Bridges{}.type', '{2}.Bridges{0}.length')| fields " + + " c | head 1", + TEST_INDEX_PEOPLE2, candidate, candidate, candidate, candidate)); + + verifySchema(actual, + schema("c", "string")); + + verifyDataRows( + actual, + rows( + + "[[\"Bridge of Sighs\",\"Ponte della Paglia\"],8981.0]")); + } + @Test public void testJsonKeys() { JSONObject actual = From a2cf460f8685253365ec4b415823910abc888b6e Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 16 May 2025 15:37:59 +0800 Subject: [PATCH 28/50] apply spotles Signed-off-by: xinyual --- .../jsonUDF/JsonExtractFunctionImpl.java | 24 +++--- .../CalcitePPLJsonBuiltinFunctionIT.java | 82 +++++++++---------- 2 files changed, 51 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index d55f6f52a1c..e6a6415d929 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -10,7 +10,6 @@ import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; -import com.fasterxml.jackson.databind.JsonNode; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -59,21 +58,22 @@ public static Object eval(Object... args) { String jsonStr = (String) args[0]; List jsonPaths = Arrays.asList(args).subList(1, args.length); List pathSpecs = - jsonPaths.stream() - .map(Object::toString) - .map(JsonUtils::convertToJsonPath) - .map(s -> " lax " + s).toList(); + jsonPaths.stream() + .map(Object::toString) + .map(JsonUtils::convertToJsonPath) + .map(s -> " lax " + s) + .toList(); List results = new ArrayList<>(); for (String pathSpec : pathSpecs) { Object queryResult = a.jsonQuery(jsonStr, pathSpec, WITHOUT_ARRAY, NULL, NULL, false); Object valueResult = - a.jsonValue( - jsonStr, - pathSpec, - SqlJsonValueEmptyOrErrorBehavior.NULL, - null, - SqlJsonValueEmptyOrErrorBehavior.NULL, - null); + a.jsonValue( + jsonStr, + pathSpec, + SqlJsonValueEmptyOrErrorBehavior.NULL, + null, + SqlJsonValueEmptyOrErrorBehavior.NULL, + null); results.add(queryResult != null ? queryResult : valueResult); } if (jsonPaths.size() == 1) { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 617a5d1d0c0..44e5c087f3a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -13,7 +13,6 @@ import java.io.IOException; import java.util.List; import org.json.JSONObject; -import org.junit.Ignore; import org.junit.jupiter.api.Test; public class CalcitePPLJsonBuiltinFunctionIT extends CalcitePPLIntegTestCase { @@ -146,14 +145,16 @@ public void testJsonExtract() { executeQuery( String.format( "source=%s | head 1 | eval a = json_extract('%s', '{}'), b= json_extract('%s'," - + " '{2}.Bridges{0}.length'), c=json_extract('%s', '{}.Bridges{}.type'), d=json_extract('%s', '{2}.Bridges{0}')| fields a," - + " b,c, d | head 1", + + " '{2}.Bridges{0}.length'), c=json_extract('%s', '{}.Bridges{}.type')," + + " d=json_extract('%s', '{2}.Bridges{0}')| fields a, b,c, d | head 1", TEST_INDEX_PEOPLE2, candidate, candidate, candidate, candidate)); - verifySchema(actual, schema("a", "string"), - schema("b", "string"), - schema("c", "string"), - schema("d", "string")); + verifySchema( + actual, + schema("a", "string"), + schema("b", "string"), + schema("c", "string"), + schema("d", "string")); verifyDataRows( actual, @@ -167,45 +168,40 @@ public void testJsonExtract() { @Test public void testJsonExtractWithMultiplyResult() { String candidate = - "[\n" - + "{\n" - + "\"name\":\"London\",\n" - + "\"Bridges\":[\n" - + "{\"name\":\"Tower Bridge\",\"length\":801.0},\n" - + "{\"name\":\"Millennium Bridge\",\"length\":1066.0}\n" - + "]\n" - + "},\n" - + "{\n" - + "\"name\":\"Venice\",\n" - + "\"Bridges\":[\n" - + "{\"name\":\"Rialto Bridge\",\"length\":157.0},\n" - + "{\"type\":\"Bridge of Sighs\",\"length\":36.0},\n" - + "{\"type\":\"Ponte della Paglia\"}\n" - + "]\n" - + "},\n" - + "{\n" - + "\"name\":\"San Francisco\",\n" - + "\"Bridges\":[\n" - + "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0},\n" - + "{\"name\":\"Bay Bridge\",\"length\":23556.0}\n" - + "]\n" - + "}\n" - + "]"; + "[\n" + + "{\n" + + "\"name\":\"London\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Tower Bridge\",\"length\":801.0},\n" + + "{\"name\":\"Millennium Bridge\",\"length\":1066.0}\n" + + "]\n" + + "},\n" + + "{\n" + + "\"name\":\"Venice\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Rialto Bridge\",\"length\":157.0},\n" + + "{\"type\":\"Bridge of Sighs\",\"length\":36.0},\n" + + "{\"type\":\"Ponte della Paglia\"}\n" + + "]\n" + + "},\n" + + "{\n" + + "\"name\":\"San Francisco\",\n" + + "\"Bridges\":[\n" + + "{\"name\":\"Golden Gate Bridge\",\"length\":8981.0},\n" + + "{\"name\":\"Bay Bridge\",\"length\":23556.0}\n" + + "]\n" + + "}\n" + + "]"; JSONObject actual = - executeQuery( - String.format( - "source=%s | head 1 | eval c=json_extract('%s', '{}.Bridges{}.type', '{2}.Bridges{0}.length')| fields " - + " c | head 1", - TEST_INDEX_PEOPLE2, candidate, candidate, candidate, candidate)); - - verifySchema(actual, - schema("c", "string")); + executeQuery( + String.format( + "source=%s | head 1 | eval c=json_extract('%s', '{}.Bridges{}.type'," + + " '{2}.Bridges{0}.length')| fields c | head 1", + TEST_INDEX_PEOPLE2, candidate, candidate, candidate, candidate)); - verifyDataRows( - actual, - rows( + verifySchema(actual, schema("c", "string")); - "[[\"Bridge of Sighs\",\"Ponte della Paglia\"],8981.0]")); + verifyDataRows(actual, rows("[[\"Bridge of Sighs\",\"Ponte della Paglia\"],8981.0]")); } @Test From 3662661ca6d8a92ac02cbc113dc9e262d16fdf06 Mon Sep 17 00:00:00 2001 From: xinyual Date: Mon, 19 May 2025 10:59:33 +0800 Subject: [PATCH 29/50] add more IT Signed-off-by: xinyual --- .../jsonUDF/JsonAppendFunctionImpl.java | 6 ++-- .../jsonUDF/JsonExtendFunctionImpl.java | 9 +++--- .../function/jsonUDF/JsonUtils.java | 7 +++++ integ-test/build.gradle | 4 +-- .../CalcitePPLJsonBuiltinFunctionIT.java | 28 +++++++++++++++++++ 5 files changed, 46 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java index 38aa5ebbfcb..f3a2fd41634 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -68,8 +68,10 @@ public static Object eval(Object... args) throws JsonProcessingException { for (String expandedPath : expandedPaths) { expands.add( expandedPath - + ".meaninglessKey"); // We add meaningless Key since calcite json_insert can only - // insert when the path point to null + + MEANING_LESS_KEY_FOR_APPEND_AND_EXTEND); // We add meaningless Key since calcite + // json_insert can only + // insert when the path point to null see: + // https://github.com/apache/calcite/blob/d96709c4cc7ca962601317d0a70914ad95e306e1/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java#L737 expands.add(keys.get(i + 1)); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java index e6b97edbd57..72aaff5992f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -67,17 +67,18 @@ public static Object eval(Object... args) throws JsonProcessingException { } } else if (value instanceof String stringValue) { try { - List targetValues = gson.fromJson(stringValue, List.class); + List targetValues = + gson.fromJson(stringValue, List.class); // We first try to extend it as an array for (Object targetValue : targetValues) { - expands.add(expandedPath + ".meaninglessKey"); + expands.add(expandedPath + MEANING_LESS_KEY_FOR_APPEND_AND_EXTEND); expands.add(targetValue); } } catch (Exception e) { - expands.add(expandedPath + ".meaninglessKey"); + expands.add(expandedPath + MEANING_LESS_KEY_FOR_APPEND_AND_EXTEND); expands.add(value); } } else { - expands.add(expandedPath + ".meaninglessKey"); + expands.add(expandedPath + MEANING_LESS_KEY_FOR_APPEND_AND_EXTEND); expands.add(value); } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java index 241178acfda..24eff144a9d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java @@ -14,6 +14,7 @@ public class JsonUtils { static ObjectMapper objectMapper = new ObjectMapper(); public static Gson gson = new Gson(); + public static String MEANING_LESS_KEY_FOR_APPEND_AND_EXTEND = ".meaningless_key"; /** * @param input candidate json path like a.b{}.c{2} @@ -53,6 +54,12 @@ public static String convertToJsonPath(String input) { return sb.toString(); } + /** + * Transfer the object input to json node + * + * @param input + * @return + */ public static JsonNode convertInputToJsonNode(Object input) { try { JsonNode root; diff --git a/integ-test/build.gradle b/integ-test/build.gradle index f164db33f3a..7b5bade79ce 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -448,8 +448,8 @@ integTest { dependsOn ':opensearch-sql-plugin:bundlePlugin' if(getOSFamilyType() != "windows") { - //dependsOn startPrometheus - //finalizedBy stopPrometheus + dependsOn startPrometheus + finalizedBy stopPrometheus } // enable calcite codegen in IT diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 44e5c087f3a..8aee89707f4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -247,6 +247,34 @@ public void testJsonSet() { verifyDataRows(actual, rows("{\"a\":[{\"b\":\"3\"},{\"b\":\"3\"}]}")); } + @Test + public void testJsonSetWithWrongPath() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =json_set('{\"a\":[{\"b\":1},{\"b\":2}]}', 'a{}.b.d', '3')|" + + " fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows("{\"a\":[{\"b\":1},{\"b\":2}]}")); + } + + @Test + public void testJsonSetPartialSet() { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval a =json_set('{\"a\":[{\"b\":1},{\"b\":{\"c\": 2}}]}', 'a{}.b.c'," + + " '3')| fields a | head 1", + TEST_INDEX_PEOPLE2)); + + verifySchema(actual, schema("a", "string")); + + verifyDataRows(actual, rows("{\"a\":[{\"b\":1},{\"b\":{\"c\":\"3\"}}]}")); + } + @Test public void testJsonDelete() { JSONObject actual = From 50738d6bffbc635c6e043487a4fd13600f9dd3bf Mon Sep 17 00:00:00 2001 From: xinyual Date: Mon, 19 May 2025 11:02:29 +0800 Subject: [PATCH 30/50] remove to json string Signed-off-by: xinyual --- .../function/PPLBuiltinOperators.java | 3 -- .../expression/function/PPLFuncImpTable.java | 1 - .../jsonUDF/ToJsonStringFunctionImpl.java | 52 ------------------- integ-test/build.gradle | 4 +- .../CalcitePPLJsonBuiltinFunctionIT.java | 14 ----- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 - ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 - 7 files changed, 2 insertions(+), 74 deletions(-) delete mode 100644 core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 787acc5b4a7..642fa169243 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -34,7 +34,6 @@ import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl; -import org.opensearch.sql.expression.function.jsonUDF.ToJsonStringFunctionImpl; import org.opensearch.sql.expression.function.udf.CryptographicFunction; import org.opensearch.sql.expression.function.udf.datetime.AddSubDateFunction; import org.opensearch.sql.expression.function.udf.datetime.CurrentFunction; @@ -68,8 +67,6 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { // Math functions public static final SqlOperator SPAN = new SpanFunctionImpl().toUDF("SPAN"); public static final SqlOperator JSON = new JsonFunctionImpl().toUDF("JSON"); - public static final SqlOperator TO_JSON_STRING = - new ToJsonStringFunctionImpl().toUDF("TO_JSON_STRING"); public static final SqlOperator JSON_ARRAY_LENGTH = new JsonArrayLengthFunctionImpl().toUDF("JSON_ARRAY_LENGTH"); public static final SqlOperator JSON_EXTRACT = diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 67d51be24db..ca0e31c780f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -319,7 +319,6 @@ void populate() { Stream.concat(Stream.of(builder.makeFlag(NULL_ON_NULL)), Arrays.stream(args)) .toArray(RexNode[]::new)))); registerOperator(JSON, PPLBuiltinOperators.JSON); - registerOperator(TO_JSON_STRING, PPLBuiltinOperators.TO_JSON_STRING); registerOperator(JSON_ARRAY_LENGTH, PPLBuiltinOperators.JSON_ARRAY_LENGTH); registerOperator(JSON_EXTRACT, PPLBuiltinOperators.JSON_EXTRACT); registerOperator(JSON_KEYS, PPLBuiltinOperators.JSON_KEYS); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java deleted file mode 100644 index ff141656918..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/ToJsonStringFunctionImpl.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.function.jsonUDF; - -import static org.opensearch.sql.calcite.utils.PPLReturnTypes.STRING_FORCE_NULLABLE; - -import java.util.List; -import java.util.Map; -import org.apache.calcite.adapter.enumerable.NotNullImplementor; -import org.apache.calcite.adapter.enumerable.NullPolicy; -import org.apache.calcite.adapter.enumerable.RexImpTable; -import org.apache.calcite.adapter.enumerable.RexToLixTranslator; -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Types; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.runtime.JsonFunctions; -import org.apache.calcite.schema.impl.ScalarFunctionImpl; -import org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.opensearch.sql.expression.function.ImplementorUDF; - -public class ToJsonStringFunctionImpl extends ImplementorUDF { - public ToJsonStringFunctionImpl() { - super(new ToJsonStringImplementor(), NullPolicy.ANY); - } - - @Override - public SqlReturnTypeInference getReturnTypeInference() { - return STRING_FORCE_NULLABLE; - } - - public static class ToJsonStringImplementor implements NotNullImplementor { - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - ScalarFunctionImpl function = - (ScalarFunctionImpl) - ScalarFunctionImpl.create( - Types.lookupMethod(ToJsonStringFunctionImpl.class, "eval", Object[].class)); - return function.getImplementor().implement(translator, call, RexImpTable.NullAs.NULL); - } - } - - public static Object eval(Object... args) { - if (args[0] instanceof Map) { - return JsonFunctions.jsonize(args[0]); - } - return null; - } -} diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 7b5bade79ce..f164db33f3a 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -448,8 +448,8 @@ integTest { dependsOn ':opensearch-sql-plugin:bundlePlugin' if(getOSFamilyType() != "windows") { - dependsOn startPrometheus - finalizedBy stopPrometheus + //dependsOn startPrometheus + //finalizedBy stopPrometheus } // enable calcite codegen in IT diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java index 8aee89707f4..99623c39410 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLJsonBuiltinFunctionIT.java @@ -85,20 +85,6 @@ public void testJsonArrayWithDifferentType() { verifyDataRows(actual, rows("[1,\"123\",\"{\\\"name\\\":3}\"]")); } - @Test - public void testToJsonString() { - JSONObject actual = - executeQuery( - String.format( - "source=%s | eval a = to_json_string(name) | fields a" + " | head 1", - TEST_INDEX_GAME_OF_THRONES)); - - verifySchema(actual, schema("a", "string")); - - verifyDataRows( - actual, rows("{\"firstname\":\"Daenerys\",\"lastname\":\"Targaryen\",\"ofHerName\":1}")); - } - @Test public void testJsonArrayLength() { JSONObject actual = diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 164e91c2140..d792b5fa212 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -372,7 +372,6 @@ JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; JSON_OBJECT: 'JSON_OBJECT'; JSON_ARRAY: 'JSON_ARRAY'; -TO_JSON_STRING: 'TO_JSON_STRING'; JSON_ARRAY_LENGTH: 'JSON_ARRAY_LENGTH'; JSON_EXTRACT: 'JSON_EXTRACT'; JSON_KEYS: 'JSON_KEYS'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 661ca6c5b6c..71942add7fb 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -702,7 +702,6 @@ jsonFunctionName : JSON | JSON_OBJECT | JSON_ARRAY - | TO_JSON_STRING | JSON_ARRAY_LENGTH | JSON_EXTRACT | JSON_KEYS From 6621bf35f810640dd4eb8df3016d29ca305127f3 Mon Sep 17 00:00:00 2001 From: xinyual Date: Mon, 19 May 2025 11:05:43 +0800 Subject: [PATCH 31/50] remove useless change Signed-off-by: xinyual --- integ-test/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index f164db33f3a..7b5bade79ce 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -448,8 +448,8 @@ integTest { dependsOn ':opensearch-sql-plugin:bundlePlugin' if(getOSFamilyType() != "windows") { - //dependsOn startPrometheus - //finalizedBy stopPrometheus + dependsOn startPrometheus + finalizedBy stopPrometheus } // enable calcite codegen in IT From f444398041e0c0a5c1d2735b55f50557db349bd8 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 22 May 2025 17:20:40 +0800 Subject: [PATCH 32/50] add UT Signed-off-by: xinyual --- .../src/main/antlr/OpenSearchPPLParser.g4 | 1 - .../function/BuiltinFunctionName.java | 1 - .../jsonUDF/JsonAppendFunctionImpl.java | 63 ------------------- .../jsonUDF/JsonExtractFunctionImpl.java | 6 +- .../function/jsonUDF/JsonUtils.java | 3 +- .../expression/json/JsonFunctionsTest.java | 56 +++++++++++++++++ 6 files changed, 59 insertions(+), 71 deletions(-) diff --git a/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 b/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 index 133cf64be58..0e1cf0db946 100644 --- a/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 +++ b/async-query-core/src/main/antlr/OpenSearchPPLParser.g4 @@ -868,7 +868,6 @@ jsonFunctionName | JSON_OBJECT | JSON_ARRAY | JSON_ARRAY_LENGTH - | TO_JSON_STRING | JSON_EXTRACT | JSON_KEYS | JSON_VALID diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index f59c705ae01..0d2e29be43e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -227,7 +227,6 @@ public enum BuiltinFunctionName { JSON(FunctionName.of("json")), JSON_OBJECT(FunctionName.of("json_object")), JSON_ARRAY(FunctionName.of("json_array")), - TO_JSON_STRING(FunctionName.of("to_json_string")), JSON_ARRAY_LENGTH(FunctionName.of("json_array_length")), JSON_EXTRACT(FunctionName.of("json_extract")), JSON_KEYS(FunctionName.of("json_keys")), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java index f3a2fd41634..32980b1659a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -10,13 +10,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.jayway.jsonpath.Configuration; -import com.jayway.jsonpath.DocumentContext; -import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.PathNotFoundException; -import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; -import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -77,60 +70,4 @@ public static Object eval(Object... args) throws JsonProcessingException { } return JsonFunctions.jsonInsert(jsonStr, expands.toArray()); } - - public static String jsonAppendIfArray(Object json, List pathValueMap, boolean isExtend) { - try { - JsonNode tree = convertInputToJsonNode(json); - - Configuration conf = - Configuration.builder() - .jsonProvider(new JacksonJsonNodeJsonProvider()) - .mappingProvider(new JacksonMappingProvider()) - .build(); - - DocumentContext context = JsonPath.using(conf).parse(tree); - - for (int index = 0; index < pathValueMap.size(); index += 2) { - String jsonPath = pathValueMap.get(index).toString(); - Object valueToAppend = pathValueMap.get(index + 1); - JsonNode targets; - try { - targets = context.read(jsonPath); - } catch (PathNotFoundException e) { - continue; - } - if (JsonPath.isPathDefinite(jsonPath)) { - if (targets instanceof ArrayNode arrayNode) { - if (isExtend && valueToAppend instanceof List list) { - for (Object value : list) { - arrayNode.addPOJO(value); - } - } else { - arrayNode.addPOJO(valueToAppend); - } - } - } else { - // Some * inside. an arrayNode returned - for (int i = 0; i < targets.size(); i++) { - JsonNode target = targets.get(i); - if (target instanceof ArrayNode arrayNode) { - if (isExtend && valueToAppend instanceof List list) { - for (Object value : list) { - arrayNode.addPOJO(value); - } - } else { - arrayNode.addPOJO(valueToAppend); - } - } - } - } - } - return tree.toString(); - } catch (Exception e) { - if (e instanceof PathNotFoundException) { - return json.toString(); - } - throw new RuntimeException("Failed to process JSON", e); - } - } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index e6a6415d929..c56e1b77c39 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -58,11 +58,7 @@ public static Object eval(Object... args) { String jsonStr = (String) args[0]; List jsonPaths = Arrays.asList(args).subList(1, args.length); List pathSpecs = - jsonPaths.stream() - .map(Object::toString) - .map(JsonUtils::convertToJsonPath) - .map(s -> " lax " + s) - .toList(); + jsonPaths.stream().map(Object::toString).map(JsonUtils::convertToJsonPath).toList(); List results = new ArrayList<>(); for (String pathSpec : pathSpecs) { Object queryResult = a.jsonQuery(jsonStr, pathSpec, WITHOUT_ARRAY, NULL, NULL, false); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java index 24eff144a9d..da8dc2a2413 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonUtils.java @@ -30,7 +30,8 @@ public static String convertToJsonPath(String input) { if (c == '{') { int end = input.indexOf('}', i); - if (end == -1) throw new IllegalArgumentException("Unmatched { in input"); + if (end == -1) + throw new IllegalArgumentException("Unmatched { in input when converting json path"); String index = input.substring(i + 1, end).trim(); if (index.isEmpty()) { diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index bba8475c110..5dab2e25adf 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -12,7 +12,9 @@ import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; +import com.fasterxml.jackson.databind.JsonNode; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -34,6 +36,7 @@ import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.LiteralExpression; +import org.opensearch.sql.expression.function.jsonUDF.JsonUtils; @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { @@ -217,4 +220,57 @@ void json_returnsSemanticCheckException() { () -> DSL.castJson(expr).valueOf(), "Expected to throw SemanticCheckException when calling castJson with " + expr)); } + + @Test + void test_convertToJsonPath() { + List originalJsonPath = List.of("{}", "a.b.c", "a{2}.c", "{3}.bc{}.d{1}"); + List targetJsonPath = List.of("$.[*]", "$.a.b.c", "$.a[2].c", "$.[3].bc[*].d[1]"); + List convertedJsonPath = + originalJsonPath.stream().map(JsonUtils::convertToJsonPath).toList(); + assertEquals(targetJsonPath, convertedJsonPath); + } + + @Test + void test_convertToJsonPathWithWrongPath() { + IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> convertToJsonPath("a.{")); + assertEquals(e.getMessage(), "Unmatched { in input when converting json path"); + } + + @Test + void test_jsonPathExpand() { + String jsonStr = + "{\"a\": {\"b\": {\"c\": 1}}, \"a2\": [{\"b2\": [{\"c2\": 1}, {\"c2\": 2}]}, {\"b2\":" + + " [{\"c2\": 1}, {\"c2\": 2}]}, {\"b2\": [{\"c2\": 1}]}], \"a3\": [{\"b3\": [{\"c2\":" + + " 1}, {\"c2\": 2}]}, {\"b4\": [{\"c2\": 1}, {\"c2\": 2}]}, {\"b5\": [{\"c2\": 1}," + + " {\"c2\": 2}]}]}"; + JsonNode node = convertInputToJsonNode(jsonStr); + String candidate1 = "$.a.b.c"; + List target1 = List.of("$.a.b.c"); + assertEquals(expandJsonPath(node, candidate1), target1); + String candidate2 = "$.a2[*].b2[*].c2"; + List target2 = + List.of( + "$.a2[0].b2[0].c2", + "$.a2[0].b2[1].c2", + "$.a2[1].b2[0].c2", + "$.a2[1].b2[1].c2", + "$.a2[2].b2[0].c2"); + assertEquals(expandJsonPath(node, candidate2), target2); + String candidate3 = "$.a3[*].b3[*].c2"; + List target3 = List.of("$.a3[0].b3[0].c2", "$.a3[0].b3[1].c2"); + assertEquals(expandJsonPath(node, candidate3), target3); + String candidate4 = "$.a2[*].b2[1].c2"; + List target4 = List.of("$.a2[0].b2[1].c2", "$.a2[1].b2[1].c2", "$.a2[2].b2[1]"); + assertEquals(expandJsonPath(node, candidate4), target4); + } + + @Test + void test_jsonPathExpandAtArray() { + String jsonStr = "[{\"c\": 1}, {\"c\": 1}, {\"c\": 1}]"; + JsonNode node = convertInputToJsonNode(jsonStr); + String candidate1 = "$.[*]"; + List target1 = List.of("$.[0]", "$.[1]", "$.[2]"); + assertEquals(expandJsonPath(node, candidate1), target1); + } } From e5fc7a58438855dc8fe0e65f9987b5c1a12df39e Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 22 May 2025 19:59:28 +0800 Subject: [PATCH 33/50] add some doc Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 108 ++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 2 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 77d9d00f45f..6685b96f8a5 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -41,11 +41,11 @@ JSON Description >>>>>>>>>>> -Usage: `json(value)` Evaluates whether a string can be parsed as a json-encoded string and casted as an expression. Returns the JSON value if valid, null otherwise. +Usage: `json(value)` Evaluates whether a string can be parsed as a json-encoded string. Returns the value if valid, null otherwise. Argument type: STRING -Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY +Return type: STRING Example:: @@ -60,3 +60,107 @@ Example:: | json scalar string | "abc" | "abc" | | json empty string | | null | +---------------------+---------------------------------+-------------------------+ + +JSON_OBJECT +---------- + +Description +>>>>>>>>>>> + +Usage: `json_object(key1, value1, key2, value2...)` create a json object string with key value pairs. The key must be string. + +Argument type: key1: STRING, value1: ANY, key2: STRING, value2: ANY ... + +Return type: STRING + +Example:: + + > source=json_test | eval test_json = json_object('key', 123.45) | head 1 | fields test_json + fetched rows / total rows = 1/1 + +-------------------------+ + | test_json | + |-------------------------| + | {"key":123.45} | + +-------------------------+ + +JSON_ARRAY +---------- + +Description +>>>>>>>>>>> + +Usage: `json_array(element1, element2, ...)` create a json array string with elements. + +Argument type: element1: ANY, element2: ANY ... + +Return type: STRING + +Example:: + + > source=json_test | eval test_json_array = json_array('key', 123.45) | head 1 | fields test_json_array + fetched rows / total rows = 1/1 + +-------------------------+ + | test_json_array | + |-------------------------| + | ["key",123.45] | + +-------------------------+ + +JSON_ARRAY_LENGTH +---------- + +Description +>>>>>>>>>>> + +Usage: `json_array_length(value)` parse the string to json array and return size, if can't be parsed, return null + +Argument type: value: STRING + +Return type: INTEGER + +Example:: + + > source=json_test | eval array_length = json_array_length("[1,2,3]") | head 1 | fields array_length + fetched rows / total rows = 1/1 + +-------------------------+ + | array_length | + |-------------------------| + | 3 | + +-------------------------+ + + > source=json_test | eval array_length = json_array_length("{\"1\": 2}") | head 1 | fields array_length + fetched rows / total rows = 1/1 + +-------------------------+ + | array_length | + |-------------------------| + | null | + +-------------------------+ + +JSON_EXTRACT +---------- + +Description +>>>>>>>>>>> + +Usage: `json_extract(json_string, path1, path2, ...)` it first transfer json_string to json, then extract value using paths. If only one path, return the value, otherwise, return the list of values. If one path cannot find value, return null as the result for this path. The path use "{}" to represent index for array, "{}" means "{*}". + +Argument type: json_string: STRING, path1: STRING, path2: STRING ... + +Return type: STRING + +Example:: + + > source=json_test | eval extract = json_extract('{\"a\": [{\"b\": 1}, {\"b\": 2}]}', 'a{}.b') | head 1 | fields extract + fetched rows / total rows = 1/1 + +-------------------------+ + | test_json_array | + |-------------------------| + | [1,2] | + +-------------------------+ + + > source=json_test | eval extract = json_extract('{\"a\": [{\"b\": 1}, {\"b\": 2}]}', 'a{}.b', 'a{}'') | head 1 | fields extract + fetched rows / total rows = 1/1 + +---------------------------------+ + | test_json_array | + |---------------------------------| + | [[1,2],[{\"b\": 1}, {\"b\": 2}]]| + +---------------------------------+ \ No newline at end of file From af2ace553da91184d03e404f02b31ea9c91f5f54 Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 23 May 2025 16:38:33 +0800 Subject: [PATCH 34/50] add docs Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 178 +++++++++++++++++++++++++++++-- 1 file changed, 172 insertions(+), 6 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 6685b96f8a5..9e2e12fac9e 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -113,7 +113,7 @@ Description Usage: `json_array_length(value)` parse the string to json array and return size, if can't be parsed, return null -Argument type: value: STRING +Argument type: value: A JSON STRING Return type: INTEGER @@ -141,7 +141,7 @@ JSON_EXTRACT Description >>>>>>>>>>> -Usage: `json_extract(json_string, path1, path2, ...)` it first transfer json_string to json, then extract value using paths. If only one path, return the value, otherwise, return the list of values. If one path cannot find value, return null as the result for this path. The path use "{}" to represent index for array, "{}" means "{*}". +Usage: `json_extract(json_string, path1, path2, ...)` Extracts values using the specified JSON paths. If only one path is provided, it returns a single value. If multiple paths are provided, it returns a JSON Array in the order of the paths. If one path cannot find value, return null as the result for this path. The path use "{}" to represent index for array, "{}" means "{*}". Argument type: json_string: STRING, path1: STRING, path2: STRING ... @@ -149,7 +149,7 @@ Return type: STRING Example:: - > source=json_test | eval extract = json_extract('{\"a\": [{\"b\": 1}, {\"b\": 2}]}', 'a{}.b') | head 1 | fields extract + > source=json_test | eval extract = json_extract('{"a": [{"b": 1}, {"b": 2}]}', 'a{}.b') | head 1 | fields extract fetched rows / total rows = 1/1 +-------------------------+ | test_json_array | @@ -157,10 +157,176 @@ Example:: | [1,2] | +-------------------------+ - > source=json_test | eval extract = json_extract('{\"a\": [{\"b\": 1}, {\"b\": 2}]}', 'a{}.b', 'a{}'') | head 1 | fields extract + > source=json_test | eval extract = json_extract('{"a": [{"b": 1}, {"b": 2}]}', 'a{}.b', 'a{}'') | head 1 | fields extract fetched rows / total rows = 1/1 +---------------------------------+ | test_json_array | |---------------------------------| - | [[1,2],[{\"b\": 1}, {\"b\": 2}]]| - +---------------------------------+ \ No newline at end of file + | [[1,2],[{"b": 1}, {"b": 2}]] | + +---------------------------------+ + +JSON_DELETE +---------- + +Description +>>>>>>>>>>> + +Usage: `json_delete(json_string, path1, path2, ...)` Delete values using the specified JSON paths. Return the json string after deleting. If one path cannot find value, do nothing. + +Argument type: json_string: STRING, path1: STRING, path2: STRING ... + +Return type: STRING + +Example:: + + > source=json_test | eval delete = json_delete('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b') | head 1 | fields delete + fetched rows / total rows = 1/1 + +-------------------------+ + | delete | + |-------------------------| + | {"a": [{"b": 1}]} | + +-------------------------+ + + > source=json_test | eval delete = json_delete('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', 'a{1}.b') | head 1 | fields delete + fetched rows / total rows = 1/1 + +-------------------------+ + | delete | + |-------------------------| + | {"a": []} | + +-------------------------+ + + > source=json_test | eval delete = json_delete('{"a": [{"b": 1}, {"b": 2}]}', 'a{2}.b') | head 1 | fields delete + fetched rows / total rows = 1/1 + +------------------------------+ + | delete | + |------------------------------| + | {"a": [{"b": 1}, {"b": 2}]} | + +------------------------------+ + +JSON_SET +---------- + +Description +>>>>>>>>>>> + +Usage: `json_set(json_string, path1, value1, path2, value2...)` Set values to corresponding paths using the specified JSON paths. If one path's parent node is not a json object, skip the path. Return the json string after setting. + +Argument type: json_string: STRING, path1: STRING, value1: ANY, path2: STRING, value2: ANY ... + +Return type: STRING + +Example:: + + > source=json_test | eval jsonSet = json_set('{"a": [{"b": 1}]}', 'a{0}.b', 3) | head 1 | fields jsonSet + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonSet | + |-------------------------| + | {"a": [{"b": 3}]} | + +-------------------------+ + + > source=json_test | eval jsonSet = json_set('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', 3, 'a{1}.b', 4) | head 1 | fields jsonSet + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonSet | + |-------------------------| + | {"a": [{"b": 3}]} | + +-------------------------+ + +JSON_APPEND +---------- + +Description +>>>>>>>>>>> + +Usage: `json_append(json_string, path1, value1, path2, value2...)` Append values to corresponding paths using the specified JSON paths. If one path's target node is not an array, skip the path. Return the json string after setting. + +Argument type: json_string: STRING, path1: STRING, value1: ANY, path2: STRING, value2: ANY ... + +Return type: STRING + +Example:: + + > source=json_test | eval jsonAppend = json_set('{"a": [{"b": 1}]}', 'a', 3) | head 1 | fields jsonAppend + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonAppend | + |-------------------------| + | {"a": [{"b": 1}, 3]} | + +-------------------------+ + + > source=json_test | eval jsonAppend = json_append('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', 3, 'a{1}.b', 4) | head 1 | fields jsonAppend + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonAppend | + |-------------------------| + | {"a": [{"b": 1}, 3]} | + +-------------------------+ + + > source=json_test | eval jsonAppend = json_append('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', '[1,2]', 'a{1}.b', 4) | head 1 | fields jsonAppend + fetched rows / total rows = 1/1 + +----------------------------+ + | jsonAppend | + |----------------------------| + | {"a": [{"b": 1}, "[1,2]"]} | + +----------------------------+ + +JSON_EXTEND +---------- + +Description +>>>>>>>>>>> + +Usage: `json_extend(json_string, path1, value1, path2, value2...)` Extend values to corresponding paths using the specified JSON paths. If one path's target node is not an array, skip the path. The function will try to parse the value as an array. If it can be parsed, extend it to the target array. Otherwise, regard the value a single one. Return the json string after setting. + +Argument type: json_string: STRING, path1: STRING, value1: ANY, path2: STRING, value2: ANY ... + +Return type: STRING + +Example:: + + > source=json_test | eval jsonExtend = json_extend('{"a": [{"b": 1}]}', 'a', 3) | head 1 | fields jsonExtend + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonExtend | + |-------------------------| + | {"a": [{"b": 1}, 3]} | + +-------------------------+ + + > source=json_test | eval jsonExtend = json_extend('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', 3, 'a{1}.b', 4) | head 1 | fields jsonExtend + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonExtend | + |-------------------------| + | {"a": [{"b": 1}, 3]} | + +-------------------------+ + + > source=json_test | eval jsonExtend = json_extend('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', '[1,2]') | head 1 | fields jsonExtend + fetched rows / total rows = 1/1 + +----------------------------+ + | jsonExtend | + |----------------------------| + | {"a": [{"b": 1},1,2]} | + +----------------------------+ + +JSON_KEYS +---------- + +Description +>>>>>>>>>>> + +Usage: `json_keys(json_string)` Return the key list of the json_string as a string if it's an object json string. Otherwise, return null. + +Argument type: json_string: A JSON STRING + +Return type: STRING + +Example:: + + > source=json_test | eval jsonKeys = json_keys('{"a": 1, "b": 2}') | head 1 | fields jsonKeys + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonKeys | + |-------------------------| + | ["a","b"] | + +-------------------------+ \ No newline at end of file From e59d8fe3785fcb72c5db32f4d136ffca49b50e67 Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 28 May 2025 15:08:53 +0800 Subject: [PATCH 35/50] fix doc Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 9e2e12fac9e..9e69fbaa7b8 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -157,7 +157,7 @@ Example:: | [1,2] | +-------------------------+ - > source=json_test | eval extract = json_extract('{"a": [{"b": 1}, {"b": 2}]}', 'a{}.b', 'a{}'') | head 1 | fields extract + > source=json_test | eval extract = json_extract('{"a": [{"b": 1}, {"b": 2}]}', 'a{}.b', 'a{}') | head 1 | fields extract fetched rows / total rows = 1/1 +---------------------------------+ | test_json_array | @@ -184,7 +184,7 @@ Example:: +-------------------------+ | delete | |-------------------------| - | {"a": [{"b": 1}]} | + | {"a": [{},{"b": 1}]} | +-------------------------+ > source=json_test | eval delete = json_delete('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', 'a{1}.b') | head 1 | fields delete @@ -227,11 +227,11 @@ Example:: > source=json_test | eval jsonSet = json_set('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', 3, 'a{1}.b', 4) | head 1 | fields jsonSet fetched rows / total rows = 1/1 - +-------------------------+ - | jsonSet | - |-------------------------| - | {"a": [{"b": 3}]} | - +-------------------------+ + +-----------------------------+ + | jsonSet | + |-----------------------------| + | {"a": [{"b": 3},{"b": 4}]} | + +-----------------------------+ JSON_APPEND ---------- @@ -263,7 +263,7 @@ Example:: | {"a": [{"b": 1}, 3]} | +-------------------------+ - > source=json_test | eval jsonAppend = json_append('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', '[1,2]', 'a{1}.b', 4) | head 1 | fields jsonAppend + > source=json_test | eval jsonAppend = json_append('{"a": [{"b": 1}]}', 'a', '[1,2]', 'a{1}.b', 4) | head 1 | fields jsonAppend fetched rows / total rows = 1/1 +----------------------------+ | jsonAppend | @@ -301,7 +301,7 @@ Example:: | {"a": [{"b": 1}, 3]} | +-------------------------+ - > source=json_test | eval jsonExtend = json_extend('{"a": [{"b": 1}, {"b": 2}]}', 'a{0}.b', '[1,2]') | head 1 | fields jsonExtend + > source=json_test | eval jsonExtend = json_extend('{"a": [{"b": 1}]}', 'a', '[1,2]') | head 1 | fields jsonExtend fetched rows / total rows = 1/1 +----------------------------+ | jsonExtend | @@ -315,7 +315,7 @@ JSON_KEYS Description >>>>>>>>>>> -Usage: `json_keys(json_string)` Return the key list of the json_string as a string if it's an object json string. Otherwise, return null. +Usage: `json_keys(json_string)` Return the key list of the Json object as a Json array. Otherwise, return null. Argument type: json_string: A JSON STRING @@ -329,4 +329,12 @@ Example:: | jsonKeys | |-------------------------| | ["a","b"] | - +-------------------------+ \ No newline at end of file + +-------------------------+ + + > source=json_test | eval jsonKeys = json_keys('{"a": {"c": 1}, "b": 2}') | head 1 | fields jsonKeys + fetched rows / total rows = 1/1 + +-------------------------+ + | jsonKeys | + |-------------------------| + | ["a","b"] | + +-------------------------+ From d30be16345ed3e26238553c0cf7ca962074740f9 Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 28 May 2025 16:19:07 +0800 Subject: [PATCH 36/50] add json path Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 9e69fbaa7b8..c0cf5d8d6cd 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -8,6 +8,13 @@ JSON Functions :local: :depth: 1 + + +JsonPath: All JSON paths used in JSON functions follow the format {}.{}..., where each represents a field name, and {} is an optional array index used only when the corresponding key refers to an array. For example, a{2}.b{0} refers to the element at index 0 of the b array, which is nested inside the element at index 2 of the a array. + +The {} notation applies only when the associated key points to an array. If {} is used without a specific index (i.e., {}), it is interpreted as a wildcard equivalent to {*}, meaning all elements in the array at that level. + + JSON_VALID ---------- From 8cc986a4103c51849b7d52bd45128487d1481b7a Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 29 May 2025 21:18:43 +0800 Subject: [PATCH 37/50] optimize doc Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index c0cf5d8d6cd..92bdfe8445a 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -10,9 +10,24 @@ JSON Functions -JsonPath: All JSON paths used in JSON functions follow the format {}.{}..., where each represents a field name, and {} is an optional array index used only when the corresponding key refers to an array. For example, a{2}.b{0} refers to the element at index 0 of the b array, which is nested inside the element at index 2 of the a array. +### JsonPath + +All JSON paths used in JSON functions follow the format: +``` +{}.{}... +``` + +Each `` represents a field name, and `{}` is an optional array index that is **only applicable** when the corresponding key refers to an array. + +For example: +`a{2}.b{0}` refers to the element at index `0` of the `b` array, which is nested inside the element at index `2` of the `a` array. + +The `{}` notation: +- Applies **only** when the associated key points to an array. +- If used as `{}`, it is interpreted as a **wildcard**, equivalent to `{*}`, meaning all elements in the array at that level. + +This syntax enables precise or broad selection of nested data structures within JSON documents. -The {} notation applies only when the associated key points to an array. If {} is used without a specific index (i.e., {}), it is interpreted as a wildcard equivalent to {*}, meaning all elements in the array at that level. JSON_VALID From 8833a3f642a5324685118c38f6200e5d5e516c2c Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 29 May 2025 21:19:39 +0800 Subject: [PATCH 38/50] optimize doc Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 92bdfe8445a..7c6609d4d79 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -10,7 +10,8 @@ JSON Functions -### JsonPath +JsonPath +-------- All JSON paths used in JSON functions follow the format: ``` From aba660f58c72aa098159914b5cc719c64bb0aee7 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 29 May 2025 21:20:16 +0800 Subject: [PATCH 39/50] optimize doc Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 7c6609d4d79..4b842310efa 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -14,9 +14,7 @@ JsonPath -------- All JSON paths used in JSON functions follow the format: -``` -{}.{}... -``` +`{}.{}...` Each `` represents a field name, and `{}` is an optional array index that is **only applicable** when the corresponding key refers to an array. From a1990662ed6c273c99f8b7c89aab2b03ff17551a Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 29 May 2025 21:22:01 +0800 Subject: [PATCH 40/50] optimize doc Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 33 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 4b842310efa..7090f5caac3 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -14,7 +14,7 @@ JsonPath -------- All JSON paths used in JSON functions follow the format: -`{}.{}...` +``{}.{}...`` Each `` represents a field name, and `{}` is an optional array index that is **only applicable** when the corresponding key refers to an array. @@ -29,32 +29,27 @@ This syntax enables precise or broad selection of nested data structures within -JSON_VALID ----------- +JSON Path +--------- Description >>>>>>>>>>> -Usage: `json_valid(json_string)` checks if `json_string` is a valid JSON-encoded string. +All JSON paths used in JSON functions follow the format `{}.{}...`. -Argument type: STRING +Each `` represents a field name. The `{}` part is optional and is only applicable when the corresponding key refers to an array. -Return type: BOOLEAN +For example:: -Example:: + a{2}.b{0} + +This refers to the element at index 0 of the `b` array, which is nested inside the element at index 2 of the `a` array. + +Notes: + +- The `{}` notation applies **only when** the associated key points to an array. +- `{}` (without a specific index) is interpreted as a **wildcard**, equivalent to `{*}`, meaning "all elements" in the array at that level. - > source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid - fetched rows / total rows = 6/6 - +---------------------+---------------------------------+----------+ - | test_name | json_string | is_valid | - |---------------------|---------------------------------|----------| - | json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True | - | json object | {"a":"1","b":"2"} | True | - | json array | [1, 2, 3, 4] | True | - | json scalar string | "abc" | True | - | json empty string | | True | - | json invalid object | {"invalid":"json", "string"} | False | - +---------------------+---------------------------------+----------+ JSON ---------- From 5cf3d9d560d5fde8afa20f910d0c6855d4eb17ea Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 29 May 2025 21:23:19 +0800 Subject: [PATCH 41/50] optimize doc Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 7090f5caac3..3a2f41233a2 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -50,7 +50,6 @@ Notes: - The `{}` notation applies **only when** the associated key points to an array. - `{}` (without a specific index) is interpreted as a **wildcard**, equivalent to `{*}`, meaning "all elements" in the array at that level. - JSON ---------- From 06100e39ad0595e6055eed2bc18d2c5c809610ce Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 29 May 2025 21:25:58 +0800 Subject: [PATCH 42/50] optimize doc Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 3a2f41233a2..561eaeaadf0 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -10,23 +10,26 @@ JSON Functions -JsonPath --------- +JSON Path +--------- + +Description +>>>>>>>>>>> + +All JSON paths used in JSON functions follow the format `{}.{}...`. -All JSON paths used in JSON functions follow the format: -``{}.{}...`` +Each `` represents a field name. The `{}` part is optional and is only applicable when the corresponding key refers to an array. -Each `` represents a field name, and `{}` is an optional array index that is **only applicable** when the corresponding key refers to an array. +For example:: -For example: -`a{2}.b{0}` refers to the element at index `0` of the `b` array, which is nested inside the element at index `2` of the `a` array. + a{2}.b{0} -The `{}` notation: -- Applies **only** when the associated key points to an array. -- If used as `{}`, it is interpreted as a **wildcard**, equivalent to `{*}`, meaning all elements in the array at that level. +This refers to the element at index 0 of the `b` array, which is nested inside the element at index 2 of the `a` array. -This syntax enables precise or broad selection of nested data structures within JSON documents. +Notes: +- The `{}` notation applies **only when** the associated key points to an array. +- `{}` (without a specific index) is interpreted as a **wildcard**, equivalent to `{*}`, meaning "all elements" in the array at that level. JSON Path From 64177d82afa708f9d196ad88559fe61fd097ee4b Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 29 May 2025 21:28:31 +0800 Subject: [PATCH 43/50] optimize doc Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 32 ++++++-------------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 561eaeaadf0..7b510f680f7 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -10,27 +10,6 @@ JSON Functions -JSON Path ---------- - -Description ->>>>>>>>>>> - -All JSON paths used in JSON functions follow the format `{}.{}...`. - -Each `` represents a field name. The `{}` part is optional and is only applicable when the corresponding key refers to an array. - -For example:: - - a{2}.b{0} - -This refers to the element at index 0 of the `b` array, which is nested inside the element at index 2 of the `a` array. - -Notes: - -- The `{}` notation applies **only when** the associated key points to an array. -- `{}` (without a specific index) is interpreted as a **wildcard**, equivalent to `{*}`, meaning "all elements" in the array at that level. - JSON Path --------- @@ -38,20 +17,21 @@ JSON Path Description >>>>>>>>>>> -All JSON paths used in JSON functions follow the format `{}.{}...`. +All JSON paths used in JSON functions follow the format ``{}.{}...``. -Each `` represents a field name. The `{}` part is optional and is only applicable when the corresponding key refers to an array. +Each ```` represents a field name. The ``{}`` part is optional and is only applicable when the corresponding key refers to an array. For example:: a{2}.b{0} -This refers to the element at index 0 of the `b` array, which is nested inside the element at index 2 of the `a` array. +This refers to the element at index 0 of the ``b`` array, which is nested inside the element at index 2 of the ``a`` array. Notes: -- The `{}` notation applies **only when** the associated key points to an array. -- `{}` (without a specific index) is interpreted as a **wildcard**, equivalent to `{*}`, meaning "all elements" in the array at that level. +1. The ``{}`` notation applies **only when** the associated key points to an array. + +2. ``{}`` (without a specific index) is interpreted as a **wildcard**, equivalent to ``{*}``, meaning "all elements" in the array at that level. JSON ---------- From 19b47a39001ea86170456458cce2c54a3b1d9e89 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 5 Jun 2025 15:59:27 +0800 Subject: [PATCH 44/50] add version and limitation to docs Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 7b510f680f7..659052aee7d 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -17,6 +17,10 @@ JSON Path Description >>>>>>>>>>> +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + All JSON paths used in JSON functions follow the format ``{}.{}...``. Each ```` represents a field name. The ``{}`` part is optional and is only applicable when the corresponding key refers to an array. @@ -39,6 +43,10 @@ JSON Description >>>>>>>>>>> +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + Usage: `json(value)` Evaluates whether a string can be parsed as a json-encoded string. Returns the value if valid, null otherwise. Argument type: STRING @@ -65,6 +73,10 @@ JSON_OBJECT Description >>>>>>>>>>> +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + Usage: `json_object(key1, value1, key2, value2...)` create a json object string with key value pairs. The key must be string. Argument type: key1: STRING, value1: ANY, key2: STRING, value2: ANY ... @@ -87,6 +99,10 @@ JSON_ARRAY Description >>>>>>>>>>> +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + Usage: `json_array(element1, element2, ...)` create a json array string with elements. Argument type: element1: ANY, element2: ANY ... @@ -109,6 +125,10 @@ JSON_ARRAY_LENGTH Description >>>>>>>>>>> +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + Usage: `json_array_length(value)` parse the string to json array and return size, if can't be parsed, return null Argument type: value: A JSON STRING @@ -139,6 +159,10 @@ JSON_EXTRACT Description >>>>>>>>>>> +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + Usage: `json_extract(json_string, path1, path2, ...)` Extracts values using the specified JSON paths. If only one path is provided, it returns a single value. If multiple paths are provided, it returns a JSON Array in the order of the paths. If one path cannot find value, return null as the result for this path. The path use "{}" to represent index for array, "{}" means "{*}". Argument type: json_string: STRING, path1: STRING, path2: STRING ... @@ -169,6 +193,10 @@ JSON_DELETE Description >>>>>>>>>>> +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + Usage: `json_delete(json_string, path1, path2, ...)` Delete values using the specified JSON paths. Return the json string after deleting. If one path cannot find value, do nothing. Argument type: json_string: STRING, path1: STRING, path2: STRING ... @@ -207,6 +235,10 @@ JSON_SET Description >>>>>>>>>>> +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + Usage: `json_set(json_string, path1, value1, path2, value2...)` Set values to corresponding paths using the specified JSON paths. If one path's parent node is not a json object, skip the path. Return the json string after setting. Argument type: json_string: STRING, path1: STRING, value1: ANY, path2: STRING, value2: ANY ... @@ -237,6 +269,10 @@ JSON_APPEND Description >>>>>>>>>>> +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + Usage: `json_append(json_string, path1, value1, path2, value2...)` Append values to corresponding paths using the specified JSON paths. If one path's target node is not an array, skip the path. Return the json string after setting. Argument type: json_string: STRING, path1: STRING, value1: ANY, path2: STRING, value2: ANY ... @@ -275,6 +311,10 @@ JSON_EXTEND Description >>>>>>>>>>> +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + Usage: `json_extend(json_string, path1, value1, path2, value2...)` Extend values to corresponding paths using the specified JSON paths. If one path's target node is not an array, skip the path. The function will try to parse the value as an array. If it can be parsed, extend it to the target array. Otherwise, regard the value a single one. Return the json string after setting. Argument type: json_string: STRING, path1: STRING, value1: ANY, path2: STRING, value2: ANY ... @@ -313,6 +353,10 @@ JSON_KEYS Description >>>>>>>>>>> +Version: 3.1.0 + +Limitation: Only works when plugins.calcite.enabled=true + Usage: `json_keys(json_string)` Return the key list of the Json object as a Json array. Otherwise, return null. Argument type: json_string: A JSON STRING From 425afc88d5ecc2d29fb97b0026005387a6947d48 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 5 Jun 2025 16:00:24 +0800 Subject: [PATCH 45/50] remove original implementation of V2 Signed-off-by: xinyual --- .../function/BuiltinFunctionRepository.java | 2 - .../sql/expression/json/JsonFunctions.java | 38 ------------------- 2 files changed, 40 deletions(-) delete mode 100644 core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java index 72d637fd2ba..79ea58b8608 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionRepository.java @@ -28,7 +28,6 @@ import org.opensearch.sql.expression.datetime.DateTimeFunctions; import org.opensearch.sql.expression.datetime.IntervalClause; import org.opensearch.sql.expression.ip.IPFunctions; -import org.opensearch.sql.expression.json.JsonFunctions; import org.opensearch.sql.expression.operator.arthmetic.ArithmeticFunctions; import org.opensearch.sql.expression.operator.arthmetic.MathematicalFunctions; import org.opensearch.sql.expression.operator.convert.TypeCastOperators; @@ -84,7 +83,6 @@ public static synchronized BuiltinFunctionRepository getInstance() { SystemFunctions.register(instance); OpenSearchFunctions.register(instance); IPFunctions.register(instance); - JsonFunctions.register(instance); } return instance; } diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java deleted file mode 100644 index 75f134aa4e9..00000000000 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.expression.json; - -import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; -import static org.opensearch.sql.data.type.ExprCoreType.STRING; -import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; -import static org.opensearch.sql.expression.function.FunctionDSL.define; -import static org.opensearch.sql.expression.function.FunctionDSL.impl; -import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; - -import lombok.experimental.UtilityClass; -import org.opensearch.sql.expression.function.BuiltinFunctionName; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.DefaultFunctionResolver; -import org.opensearch.sql.utils.JsonUtils; - -@UtilityClass -public class JsonFunctions { - public void register(BuiltinFunctionRepository repository) { - repository.register(jsonValid()); - repository.register(jsonFunction()); - } - - private DefaultFunctionResolver jsonValid() { - return define( - BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); - } - - private DefaultFunctionResolver jsonFunction() { - return define( - BuiltinFunctionName.JSON.getName(), - impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); - } -} From 10b102a443363e06ef1cee2c634c040e836a2645 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 5 Jun 2025 16:01:33 +0800 Subject: [PATCH 46/50] revert useless change Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 659052aee7d..6370b84b80e 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -17,10 +17,6 @@ JSON Path Description >>>>>>>>>>> -Version: 3.1.0 - -Limitation: Only works when plugins.calcite.enabled=true - All JSON paths used in JSON functions follow the format ``{}.{}...``. Each ```` represents a field name. The ``{}`` part is optional and is only applicable when the corresponding key refers to an array. From 0af8cc8f8805bc077d5f0f08a0368b9abcf712bb Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 5 Jun 2025 17:18:08 +0800 Subject: [PATCH 47/50] remove useless UTs Signed-off-by: xinyual --- .../expression/json/JsonFunctionsTest.java | 165 ------------------ 1 file changed, 165 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 5dab2e25adf..8159fd6c115 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -7,67 +7,22 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import static org.opensearch.sql.expression.function.jsonUDF.JsonUtils.*; import com.fasterxml.jackson.databind.JsonNode; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.sql.data.model.ExprBooleanValue; -import org.opensearch.sql.data.model.ExprCollectionValue; -import org.opensearch.sql.data.model.ExprDoubleValue; -import org.opensearch.sql.data.model.ExprIntegerValue; -import org.opensearch.sql.data.model.ExprLongValue; -import org.opensearch.sql.data.model.ExprNullValue; -import org.opensearch.sql.data.model.ExprStringValue; -import org.opensearch.sql.data.model.ExprTupleValue; -import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; -import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.function.jsonUDF.JsonUtils; @ExtendWith(MockitoExtension.class) public class JsonFunctionsTest { - @Test - public void json_valid_returns_false() { - List expressions = - List.of( - DSL.literal(LITERAL_MISSING), // missing returns false - DSL.literal(LITERAL_NULL), // null returns false - DSL.literal("invalid"), // invalid type - DSL.literal("{{[}}"), // missing bracket - DSL.literal("[}"), // missing bracket - DSL.literal("}"), // missing bracket - DSL.literal("\"missing quote"), // missing quote - DSL.literal("abc"), // not a type - DSL.literal("97ab"), // not a type - DSL.literal("{1, 2, 3, 4}"), // invalid object - DSL.literal("{\"invalid\":\"json\", \"string\"}"), // invalid object - DSL.literal("{123: 1, true: 2, null: 3}"), // invalid object - DSL.literal("[\"a\": 1, \"b\": 2]") // invalid array - ); - - expressions.stream() - .forEach( - expr -> - assertEquals( - LITERAL_FALSE, - DSL.jsonValid(expr).valueOf(), - "Expected FALSE when calling jsonValid with " + expr)); - } - @Test public void json_valid_throws_ExpressionEvaluationException() { assertThrows( @@ -75,126 +30,6 @@ public void json_valid_throws_ExpressionEvaluationException() { () -> DSL.jsonValid(DSL.literal((ExprValueUtils.booleanValue(true)))).valueOf()); } - @Test - public void json_valid_returns_true() { - - List validJsonStrings = - List.of( - // test json objects are valid - "{\"a\":\"1\",\"b\":\"2\"}", - "{\"a\":1,\"b\":{\"c\":2,\"d\":3}}", - "{\"arr1\": [1,2,3], \"arr2\": [4,5,6]}", - - // test json arrays are valid - "[1, 2, 3, 4]", - "[{\"a\":1,\"b\":2}, {\"c\":3,\"d\":2}]", - - // test json scalars are valid - "\"abc\"", - "1234", - "12.34", - "true", - "false", - "null", - - // test empty string is valid - ""); - - validJsonStrings.stream() - .forEach( - str -> - assertEquals( - LITERAL_TRUE, - DSL.jsonValid(DSL.literal(str)).valueOf(), - String.format("String %s must be valid json", str))); - } - - @Test - void json_returnsJsonObject() { - FunctionExpression exp; - - // Setup - final String objectJson = - "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " - + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; - - LinkedHashMap objectMap = new LinkedHashMap<>(); - objectMap.put("foo", new ExprStringValue("foo")); - objectMap.put("fuzz", ExprBooleanValue.of(true)); - objectMap.put("bar", new ExprLongValue(1234)); - objectMap.put("bar2", new ExprDoubleValue(12.34)); - objectMap.put("baz", ExprNullValue.of()); - objectMap.put( - "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); - objectMap.put( - "arr", - new ExprCollectionValue( - List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); - ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); - - // exercise - exp = DSL.stringToJson(DSL.literal(objectJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprTupleValue); - assertEquals(expectedTupleExpr, value); - - // also test the empty object case - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.stringToJson(DSL.literal("{}")).valueOf()); - } - - @Test - void json_returnsJsonArray() { - FunctionExpression exp; - - // Setup - final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; - ExprValue expectedArrayExpr = - new ExprCollectionValue( - List.of( - new ExprStringValue("foo"), - new ExprStringValue("fuzz"), - LITERAL_TRUE, - new ExprStringValue("bar"), - new ExprIntegerValue(1234), - new ExprDoubleValue(12.34), - LITERAL_NULL)); - - // exercise - exp = DSL.stringToJson(DSL.literal(arrayJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprCollectionValue); - assertEquals(expectedArrayExpr, value); - - // also test the empty-array case - assertEquals(new ExprCollectionValue(List.of()), DSL.stringToJson(DSL.literal("[]")).valueOf()); - } - - @Test - void json_returnsScalar() { - assertEquals( - new ExprStringValue("foobar"), DSL.stringToJson(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.stringToJson(DSL.literal("1234")).valueOf()); - - assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); - - assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); - assertEquals(LITERAL_FALSE, DSL.stringToJson(DSL.literal("false")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal(LITERAL_NULL)).valueOf()); - - assertEquals(LITERAL_MISSING, DSL.stringToJson(DSL.literal(LITERAL_MISSING)).valueOf()); - - assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("")).valueOf()); - } - @Test void json_returnsSemanticCheckException() { List expressions = From 895f24ce175b6e2c1e086f47362381c4f81bba56 Mon Sep 17 00:00:00 2001 From: xinyual Date: Thu, 5 Jun 2025 17:20:20 +0800 Subject: [PATCH 48/50] Ignore useless IT Signed-off-by: xinyual --- .../src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index f119f2e4694..fa26072e2d7 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -17,8 +17,10 @@ import java.util.Map; import org.json.JSONArray; import org.json.JSONObject; +import org.junit.Ignore; import org.junit.jupiter.api.Test; +@Ignore("https://github.com/opensearch-project/sql/issues/3565") public class JsonFunctionsIT extends PPLIntegTestCase { @Override public void init() throws Exception { From b2ba7a8f41604e71bd32b96418bbcdf92e31e711 Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 6 Jun 2025 10:05:53 +0800 Subject: [PATCH 49/50] update doc Signed-off-by: xinyual --- docs/user/ppl/functions/json.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 6370b84b80e..8357c26943a 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -125,7 +125,7 @@ Version: 3.1.0 Limitation: Only works when plugins.calcite.enabled=true -Usage: `json_array_length(value)` parse the string to json array and return size, if can't be parsed, return null +Usage: `json_array_length(value)` parse the string to json array and return size,, null is returned in case of any other valid JSON string, null or an invalid JSON. Argument type: value: A JSON STRING From dad31785653b5233d9bb62c999badedf22e0e95d Mon Sep 17 00:00:00 2001 From: xinyual Date: Fri, 6 Jun 2025 13:32:21 +0800 Subject: [PATCH 50/50] add type checker for json Signed-off-by: xinyual --- .../function/jsonUDF/JsonAppendFunctionImpl.java | 6 ++++++ .../function/jsonUDF/JsonArrayLengthFunctionImpl.java | 7 +++++++ .../function/jsonUDF/JsonDeleteFunctionImpl.java | 6 ++++++ .../function/jsonUDF/JsonExtendFunctionImpl.java | 6 ++++++ .../function/jsonUDF/JsonExtractFunctionImpl.java | 6 ++++++ .../sql/expression/function/jsonUDF/JsonFunctionImpl.java | 6 ++++++ .../expression/function/jsonUDF/JsonKeysFunctionImpl.java | 6 ++++++ .../expression/function/jsonUDF/JsonSetFunctionImpl.java | 6 ++++++ 8 files changed, 49 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java index 32980b1659a..dd76a002e06 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonAppendFunctionImpl.java @@ -24,6 +24,7 @@ import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; public class JsonAppendFunctionImpl extends ImplementorUDF { public JsonAppendFunctionImpl() { @@ -35,6 +36,11 @@ public SqlReturnTypeInference getReturnTypeInference() { return STRING_FORCE_NULLABLE; } + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + public static class JsonAppendImplementor implements NotNullImplementor { @Override public Expression implement( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java index b1281a4fa5b..cc12d8de7c5 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonArrayLengthFunctionImpl.java @@ -19,7 +19,9 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; public class JsonArrayLengthFunctionImpl extends ImplementorUDF { public JsonArrayLengthFunctionImpl() { @@ -31,6 +33,11 @@ public SqlReturnTypeInference getReturnTypeInference() { return INTEGER_FORCE_NULLABLE; } + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.STRING; + } + public static class JsonArrayLengthImplementor implements NotNullImplementor { @Override public Expression implement( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java index 1e8dcb8feac..b3a884a4f17 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonDeleteFunctionImpl.java @@ -22,6 +22,7 @@ import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; public class JsonDeleteFunctionImpl extends ImplementorUDF { public JsonDeleteFunctionImpl() { @@ -33,6 +34,11 @@ public SqlReturnTypeInference getReturnTypeInference() { return STRING_FORCE_NULLABLE; } + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + public static class JsonDeleteImplementor implements NotNullImplementor { @Override public Expression implement( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java index 72aaff5992f..dd91f1d95bd 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtendFunctionImpl.java @@ -24,6 +24,7 @@ import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; public class JsonExtendFunctionImpl extends ImplementorUDF { public JsonExtendFunctionImpl() { @@ -35,6 +36,11 @@ public SqlReturnTypeInference getReturnTypeInference() { return STRING_FORCE_NULLABLE; } + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + public static class JsonExtendImplementor implements NotNullImplementor { @Override public Expression implement( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java index c56e1b77c39..b08a2584883 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonExtractFunctionImpl.java @@ -27,6 +27,7 @@ import org.apache.calcite.sql.SqlJsonValueEmptyOrErrorBehavior; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; public class JsonExtractFunctionImpl extends ImplementorUDF { public JsonExtractFunctionImpl() { @@ -38,6 +39,11 @@ public SqlReturnTypeInference getReturnTypeInference() { return STRING_FORCE_NULLABLE; } + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + public static class JsonExtractImplementor implements NotNullImplementor { @Override public Expression implement( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java index c5095c94201..0379aeecb72 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonFunctionImpl.java @@ -18,6 +18,7 @@ import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; /** * json(value) Evaluates whether the input can be parsed as JSON format. Returns the value if valid, @@ -33,6 +34,11 @@ public SqlReturnTypeInference getReturnTypeInference() { return ReturnTypes.ARG0_FORCE_NULLABLE; } + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + public static class JsonImplementor implements NotNullImplementor { @Override public Expression implement( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java index b9ef8225fbf..40214ca7556 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonKeysFunctionImpl.java @@ -19,6 +19,7 @@ import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; public class JsonKeysFunctionImpl extends ImplementorUDF { public JsonKeysFunctionImpl() { @@ -30,6 +31,11 @@ public SqlReturnTypeInference getReturnTypeInference() { return STRING_FORCE_NULLABLE; } + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + public static class JsonKeysImplementor implements NotNullImplementor { @Override public Expression implement( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java index 4c282198177..27346b478e4 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/jsonUDF/JsonSetFunctionImpl.java @@ -23,6 +23,7 @@ import org.apache.calcite.schema.impl.ScalarFunctionImpl; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; public class JsonSetFunctionImpl extends ImplementorUDF { public JsonSetFunctionImpl() { @@ -34,6 +35,11 @@ public SqlReturnTypeInference getReturnTypeInference() { return STRING_FORCE_NULLABLE; } + @Override + public UDFOperandMetadata getOperandMetadata() { + return null; + } + public static class JsonSetImplementor implements NotNullImplementor { @Override public Expression implement(