From 9b85789818db5d9424392e992f2dfc6e0442fcfa Mon Sep 17 00:00:00 2001 From: Songkan Tang Date: Wed, 10 Dec 2025 13:54:04 +0800 Subject: [PATCH 1/3] Support sort expression pushdown for SortMergeJoin (#4830) * Support sort expression pushdown for SortMergeJoin Signed-off-by: Songkan Tang * Remove old duplicate methods in CalciteLogicalIndexScan Signed-off-by: Songkan Tang * Add more tests Signed-off-by: Songkan Tang --------- Signed-off-by: Songkan Tang --- .../sql/calcite/remote/CalciteExplainIT.java | 42 +++++++ .../sql/calcite/remote/CalcitePPLJoinIT.java | 69 +++++++++++ ...in_complex_sort_expr_pushdown_for_smj.yaml | 21 ++++ ...rt_expr_pushdown_for_smj_w_max_option.yaml | 25 ++++ .../explain_simple_sort_expr_push.json | 2 +- ...ain_simple_sort_expr_pushdown_for_smj.yaml | 28 +++++ ...ple_sort_expr_single_expr_output_push.json | 2 +- ..._sort_pass_through_join_then_pushdown.yaml | 17 +++ ...in_complex_sort_expr_pushdown_for_smj.yaml | 20 +++ ...rt_expr_pushdown_for_smj_w_max_option.yaml | 27 ++++ ...ain_simple_sort_expr_pushdown_for_smj.yaml | 21 ++++ ..._sort_pass_through_join_then_pushdown.yaml | 20 +++ .../planner/rules/SortExprIndexScanRule.java | 32 +++-- .../rules/SortProjectExprTransposeRule.java | 12 +- .../scan/AbstractCalciteIndexScan.java | 111 +++++++++++++++++ .../scan/CalciteEnumerableIndexScan.java | 6 + .../storage/scan/CalciteLogicalIndexScan.java | 115 +----------------- 17 files changed, 437 insertions(+), 133 deletions(-) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_sort_pass_through_join_then_pushdown.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml create mode 100644 integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_pass_through_join_then_pushdown.yaml diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 3f6e16c711c..d560eceb422 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -1013,6 +1013,48 @@ public void testSortComplexExprMixedWithSimpleExpr() throws Exception { assertYamlEqualsIgnoreId(expected, result); } + @Test + public void testComplexSortExprPushdownForSMJ() throws Exception { + String query = + "source=opensearch-sql_test_index_bank | rex field=lastname \\\"(?^[A-Z])\\\" |" + + " join left=a right=b on a.initial = b.firstname opensearch-sql_test_index_bank"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_complex_sort_expr_pushdown_for_smj.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testSimpleSortExprPushdownForSMJ() throws Exception { + String query = + "source=opensearch-sql_test_index_bank | join left=a right=b on a.age + 1 = b.balance - 20" + + " opensearch-sql_test_index_bank"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_simple_sort_expr_pushdown_for_smj.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testSortPassThroughJoinThenPushdown() throws Exception { + String query = + "source=opensearch-sql_test_index_bank | rex field=lastname \\\"(?^[A-Z])\\\" |" + + " join type=left left=a right=b on a.initial = b.firstname" + + " opensearch-sql_test_index_bank | sort initial"; + var result = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_sort_pass_through_join_then_pushdown.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + + @Test + public void testComplexSortExprPushdownForSMJWithMaxOption() throws Exception { + String query = + "source=opensearch-sql_test_index_bank | rex field=lastname \\\"(?^[A-Z])\\\" |" + + " join type=left max=1 lastname opensearch-sql_test_index_bank"; + var result = explainQueryYaml(query); + String expected = + loadExpectedPlan("explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml"); + assertYamlEqualsIgnoreId(expected, result); + } + @Test public void testRexExplain() throws IOException { String query = diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java index 2e19cf7ede2..986969d5085 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java @@ -970,4 +970,73 @@ public void testJoinSubsearchMaxOut() throws IOException { TEST_INDEX_STATE_COUNTRY, TEST_INDEX_OCCUPATION)); verifyNumOfRows(actual, 15); } + + @Test + public void testSimpleSortPushDownForSMJ() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | join left=a right=b on a.age + 3 = b.age - 2 %s | fields name, age," + + " b.name, b.age", + TEST_INDEX_STATE_COUNTRY, TEST_INDEX_STATE_COUNTRY)); + verifySchema( + actual, + schema("name", "string"), + schema("age", "int"), + schema("b.name", "string"), + schema("b.age", "int")); + verifyDataRows(actual, rows("Jane", 20, "John", 25), rows("John", 25, "Hello", 30)); + } + + @Test + public void testComplexSortPushDownForSMJ() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval name2=substring(name, 2, 1) | join left=a right=b on a.name2 =" + + " b.state2 [ source=%s | eval state2=substring(state, 2, 1) ] | fields name," + + " name2, b.name, b.state, state2", + TEST_INDEX_STATE_COUNTRY, TEST_INDEX_STATE_COUNTRY)); + verifySchema( + actual, + schema("name", "string"), + schema("name2", "string"), + schema("b.name", "string"), + schema("b.state", "string"), + schema("state2", "string")); + verifyDataRows( + actual, + rows("Jake", "a", "Jake", "California", "a"), + rows("Jake", "a", "David", "Washington", "a"), + rows("Jane", "a", "Jake", "California", "a"), + rows("Jane", "a", "David", "Washington", "a"), + rows("David", "a", "Jake", "California", "a"), + rows("David", "a", "David", "Washington", "a"), + rows("Hello", "e", "Hello", "New York", "e"), + rows("Peter", "e", "Hello", "New York", "e")); + } + + @Test + public void testComplexSortPushDownForSMJWithMaxOptionAndFieldList() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source=%s | eval name2=substring(name, 2, 1) | join max=1 name2,age [ source=%s |" + + " eval name2=substring(state, 2, 1) ]", + TEST_INDEX_STATE_COUNTRY, TEST_INDEX_STATE_COUNTRY)); + verifySchema( + actual, + schema("name", "string"), + schema("country", "string"), + schema("state", "string"), + schema("month", "int"), + schema("year", "int"), + schema("age", "int"), + schema("name2", "string")); + verifyDataRows( + actual, + rows("David", "USA", "Washington", 4, 2023, 40, "a"), + rows("Jake", "USA", "California", 4, 2023, 70, "a"), + rows("Hello", "USA", "New York", 4, 2023, 30, "e")); + } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj.yaml new file mode 100644 index 00000000000..4b302367ff3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$14], b.firstname=[$15], b.address=[$16], b.birthdate=[$17], b.gender=[$18], b.city=[$19], b.lastname=[$20], b.balance=[$21], b.employer=[$22], b.state=[$23], b.age=[$24], b.email=[$25], b.male=[$26]) + LogicalJoin(condition=[=($13, $15)], joinType=[inner]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($13, $15)], joinType=[inner]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=['(?^[A-Z])'], expr#14=['initial'], expr#15=[REX_EXTRACT($t6, $t13, $t14)], proj#0..12=[{exprs}], $f13=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[REX_EXTRACT($6, '(?^[A-Z])', 'initial') ASCENDING NULLS_LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC5nsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRVhfRVhUUkFDVCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAxOAogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IDcKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IDIwMDAKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":true,"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["lastname","(?^[A-Z])","initial"]}},"type":"string","order":"asc"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000, SORT->[{ + "firstname" : { + "order" : "asc", + "missing" : "_last" + } + }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":50000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"firstname":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml new file mode 100644 index 00000000000..a8754119130 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml @@ -0,0 +1,25 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$13], firstname=[$14], address=[$15], birthdate=[$16], gender=[$17], city=[$18], lastname=[$19], balance=[$20], employer=[$21], state=[$22], age=[$23], email=[$24], male=[$25]) + LogicalJoin(condition=[=($12, $19)], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], lastname=[REX_EXTRACT($6, '(?^[A-Z])', 'lastname')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalFilter(condition=[<=($13, 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _row_number_join_max_dedup_=[ROW_NUMBER() OVER (PARTITION BY $6 ORDER BY $6)]) + LogicalFilter(condition=[IS NOT NULL($6)]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..13=[{inputs}], account_number=[$t1], firstname=[$t2], address=[$t3], birthdate=[$t4], gender=[$t5], city=[$t6], lastname=[$t7], balance=[$t8], employer=[$t9], state=[$t10], age=[$t11], email=[$t12], male=[$t13]) + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($0, $7)], joinType=[left]) + EnumerableCalc(expr#0=[{inputs}], expr#1=['(?^[A-Z])'], expr#2=['lastname'], expr#3=[REX_EXTRACT($t0, $t1, $t2)], $f0=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[lastname], LIMIT->10000, SORT_EXPR->[REX_EXTRACT($0, '(?^[A-Z])', 'lastname') ASCENDING NULLS_LAST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":10000,"timeout":"1m","_source":{"includes":["lastname"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC5nsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRVhfRVhUUkFDVCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAxOQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IDgKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IDIwMDAKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":true,"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["lastname","(?^[A-Z])","lastname"]}},"type":"string","order":"asc"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableSort(sort0=[$6], dir0=[ASC]) + EnumerableCalc(expr#0..13=[{inputs}], expr#14=[1], expr#15=[<=($t13, $t14)], proj#0..12=[{exprs}], $condition=[$t15]) + EnumerableWindow(window#0=[window(partition {6} order by [6] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[IS NOT NULL($t6)], proj#0..12=[{exprs}], $condition=[$t13]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":50000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json index 8abce1ca116..0e826fc3fd2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$10], age2=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], age2=[+($10, 2)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], age=[$t0], $f1=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SORT->[{\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000, PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], age=[$t0], $f1=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age], SORT->[{\n \"age\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml new file mode 100644 index 00000000000..3c6bf5d725d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_pushdown_for_smj.yaml @@ -0,0 +1,28 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$13], b.firstname=[$14], b.address=[$15], b.birthdate=[$16], b.gender=[$17], b.city=[$18], b.lastname=[$19], b.balance=[$20], b.employer=[$21], b.state=[$22], b.age=[$23], b.email=[$24], b.male=[$25]) + LogicalJoin(condition=[=(+($10, 1), -($20, 20))], joinType=[inner]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..27=[{inputs}], proj#0..12=[{exprs}], b.account_number=[$t14], b.firstname=[$t15], b.address=[$t16], b.birthdate=[$t17], b.gender=[$t18], b.city=[$t19], b.lastname=[$t20], b.balance=[$t21], b.employer=[$t22], b.state=[$t23], b.age=[$t24], b.email=[$t25], b.male=[$t26]) + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($13, $27)], joinType=[inner]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[1], expr#14=[+($t10, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT->[{ + "age" : { + "order" : "asc", + "missing" : "_last" + } + }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"age":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=[20], expr#14=[-($t7, $t13)], proj#0..12=[{exprs}], $f13=[$t14]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000, SORT->[{ + "balance" : { + "order" : "asc", + "missing" : "_last" + } + }]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":50000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"balance":{"order":"asc","missing":"_last"}}]}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json index fe2a4af378a..722ef5379f0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_simple_sort_expr_single_expr_output_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(b=[$19])\n LogicalSort(sort0=[$19], dir0=[ASC-nulls-first])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], b=[+($7, 1)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[+($t0, $t1)], $f0=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SORT->[{\n \"balance\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000, PROJECT->[balance]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\"],\"excludes\":[]},\"sort\":[{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableCalc(expr#0=[{inputs}], expr#1=[1], expr#2=[+($t0, $t1)], $f0=[$t2])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance], SORT->[{\n \"balance\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\"],\"excludes\":[]},\"sort\":[{\"balance\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_pass_through_join_then_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_pass_through_join_then_pushdown.yaml new file mode 100644 index 00000000000..42255c0c156 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_pass_through_join_then_pushdown.yaml @@ -0,0 +1,17 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$13], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$13], dir0=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$14], b.firstname=[$15], b.address=[$16], b.birthdate=[$17], b.gender=[$18], b.city=[$19], b.lastname=[$20], b.balance=[$21], b.employer=[$22], b.state=[$23], b.age=[$24], b.email=[$25], b.male=[$26]) + LogicalJoin(condition=[=($13, $15)], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableHashJoin(condition=[=($13, $15)], joinType=[left]) + EnumerableCalc(expr#0..12=[{inputs}], expr#13=['(?^[A-Z])'], expr#14=['initial'], expr#15=[REX_EXTRACT($t6, $t13, $t14)], proj#0..12=[{exprs}], $f13=[$t15]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], SORT_EXPR->[REX_EXTRACT($6, '(?^[A-Z])', 'initial') ASCENDING NULLS_FIRST]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]},"sort":[{"_script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC5nsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRVhfRVhUUkFDVCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAxOAogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IDcKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IDIwMDAKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"MISSING_MAX":false,"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["lastname","(?^[A-Z])","initial"]}},"type":"string","order":"asc"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->50000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":50000,"timeout":"1m","_source":{"includes":["account_number","firstname","address","birthdate","gender","city","lastname","balance","employer","state","age","email","male"],"excludes":[]}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj.yaml new file mode 100644 index 00000000000..9b357fbbfed --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj.yaml @@ -0,0 +1,20 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$14], b.firstname=[$15], b.address=[$16], b.birthdate=[$17], b.gender=[$18], b.city=[$19], b.lastname=[$20], b.balance=[$21], b.employer=[$22], b.state=[$23], b.age=[$24], b.email=[$25], b.male=[$26]) + LogicalJoin(condition=[=($13, $15)], joinType=[inner]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($13, $15)], joinType=[inner]) + EnumerableSort(sort0=[$13], dir0=[ASC]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['(?^[A-Z])'], expr#20=['initial'], expr#21=[REX_EXTRACT($t6, $t19, $t20)], proj#0..12=[{exprs}], initial=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$1], dir0=[ASC]) + EnumerableLimit(fetch=[50000]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml new file mode 100644 index 00000000000..4b41b2d6c13 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_complex_sort_expr_pushdown_for_smj_w_max_option.yaml @@ -0,0 +1,27 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$13], firstname=[$14], address=[$15], birthdate=[$16], gender=[$17], city=[$18], lastname=[$19], balance=[$20], employer=[$21], state=[$22], age=[$23], email=[$24], male=[$25]) + LogicalJoin(condition=[=($12, $19)], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], lastname=[REX_EXTRACT($6, '(?^[A-Z])', 'lastname')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + LogicalFilter(condition=[<=($13, 1)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _row_number_join_max_dedup_=[ROW_NUMBER() OVER (PARTITION BY $6 ORDER BY $6)]) + LogicalFilter(condition=[IS NOT NULL($6)]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..13=[{inputs}], account_number=[$t1], firstname=[$t2], address=[$t3], birthdate=[$t4], gender=[$t5], city=[$t6], lastname=[$t7], balance=[$t8], employer=[$t9], state=[$t10], age=[$t11], email=[$t12], male=[$t13]) + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($0, $7)], joinType=[left]) + EnumerableSort(sort0=[$0], dir0=[ASC]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['(?^[A-Z])'], expr#20=['lastname'], expr#21=[REX_EXTRACT($t6, $t19, $t20)], lastname=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$6], dir0=[ASC]) + EnumerableCalc(expr#0..19=[{inputs}], expr#20=[1], expr#21=[<=($t19, $t20)], proj#0..12=[{exprs}], $condition=[$t21]) + EnumerableWindow(window#0=[window(partition {6} order by [6] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[IS NOT NULL($t6)], proj#0..18=[{exprs}], $condition=[$t19]) + EnumerableLimit(fetch=[50000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml new file mode 100644 index 00000000000..8897a1023cc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_simple_sort_expr_pushdown_for_smj.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], b.account_number=[$13], b.firstname=[$14], b.address=[$15], b.birthdate=[$16], b.gender=[$17], b.city=[$18], b.lastname=[$19], b.balance=[$20], b.employer=[$21], b.state=[$22], b.age=[$23], b.email=[$24], b.male=[$25]) + LogicalJoin(condition=[=(+($10, 1), -($20, 20))], joinType=[inner]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableCalc(expr#0..27=[{inputs}], proj#0..12=[{exprs}], b.account_number=[$t14], b.firstname=[$t15], b.address=[$t16], b.birthdate=[$t17], b.gender=[$t18], b.city=[$t19], b.lastname=[$t20], b.balance=[$t21], b.employer=[$t22], b.state=[$t23], b.age=[$t24], b.email=[$t25], b.male=[$t26]) + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($13, $27)], joinType=[inner]) + EnumerableSort(sort0=[$13], dir0=[ASC]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=[+($t10, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableSort(sort0=[$13], dir0=[ASC]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=[20], expr#20=[-($t7, $t19)], proj#0..12=[{exprs}], $f13=[$t20]) + EnumerableLimit(fetch=[50000]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_pass_through_join_then_pushdown.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_pass_through_join_then_pushdown.yaml new file mode 100644 index 00000000000..5df0b4c36a3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_sort_pass_through_join_then_pushdown.yaml @@ -0,0 +1,20 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$13], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$13], dir0=[ASC-nulls-first]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[$13], b.account_number=[$14], b.firstname=[$15], b.address=[$16], b.birthdate=[$17], b.gender=[$18], b.city=[$19], b.lastname=[$20], b.balance=[$21], b.employer=[$22], b.state=[$23], b.age=[$24], b.email=[$25], b.male=[$26]) + LogicalJoin(condition=[=($13, $15)], joinType=[left]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], initial=[REX_EXTRACT($6, '(?^[A-Z])', 'initial')]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableHashJoin(condition=[=($13, $15)], joinType=[left]) + EnumerableSort(sort0=[$13], dir0=[ASC-nulls-first]) + EnumerableCalc(expr#0..18=[{inputs}], expr#19=['(?^[A-Z])'], expr#20=['initial'], expr#21=[REX_EXTRACT($t6, $t19, $t20)], proj#0..12=[{exprs}], initial=[$t21]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableLimit(fetch=[50000]) + EnumerableCalc(expr#0..18=[{inputs}], proj#0..12=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortExprIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortExprIndexScanRule.java index aa2f8289a93..16d2da365ae 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortExprIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortExprIndexScanRule.java @@ -16,7 +16,6 @@ import org.apache.calcite.rel.RelFieldCollation.Direction; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Sort; -import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexLiteral; @@ -47,9 +46,14 @@ protected SortExprIndexScanRule(SortExprIndexScanRule.Config config) { @Override protected void onMatchImpl(RelOptRuleCall call) { - final LogicalSort sort = call.rel(0); - final LogicalProject project = call.rel(1); - final CalciteLogicalIndexScan scan = call.rel(2); + final Sort sort = call.rel(0); + final Project project = call.rel(1); + final AbstractCalciteIndexScan scan = call.rel(2); + + if (sort.getConvention() != project.getConvention() + || project.getConvention() != scan.getConvention()) { + return; + } // Only match sort - project - scan when any sort key references an expression if (!PlanUtils.sortReferencesExpr(sort, project)) { @@ -89,7 +93,7 @@ protected void onMatchImpl(RelOptRuleCall call) { return; } - CalciteLogicalIndexScan newScan; + AbstractCalciteIndexScan newScan; // If the scan's sort info already satisfies new sort, just pushdown limit if there is if (scan.isTopKPushed() && scanProvidesRequiredCollation) { newScan = scan.copy(); @@ -98,10 +102,14 @@ protected void onMatchImpl(RelOptRuleCall call) { newScan = scan.pushdownSortExpr(sortExprDigests); } + // EnumerableSort won't have limit or offset Integer limitValue = LimitIndexScanRule.extractLimitValue(sort.fetch); Integer offsetValue = LimitIndexScanRule.extractOffsetValue(sort.offset); - if (newScan != null && limitValue != null && offsetValue != null) { - newScan = (CalciteLogicalIndexScan) newScan.pushDownLimit(sort, limitValue, offsetValue); + if (newScan instanceof CalciteLogicalIndexScan && limitValue != null && offsetValue != null) { + newScan = + (CalciteLogicalIndexScan) + ((CalciteLogicalIndexScan) newScan) + .pushDownLimit((LogicalSort) sort, limitValue, offsetValue); } if (newScan != null) { @@ -125,7 +133,7 @@ protected void onMatchImpl(RelOptRuleCall call) { private List extractSortExpressionInfos( Sort sort, Project project, - CalciteLogicalIndexScan scan, + AbstractCalciteIndexScan scan, Map>> orderEquivInfoMap) { List sortExprDigests = new ArrayList<>(); @@ -161,7 +169,7 @@ private List extractSortExpressionInfos( private SortExprDigest mapThroughProject( RexNode sortKey, Project project, - CalciteLogicalIndexScan scan, + AbstractCalciteIndexScan scan, RelFieldCollation collation, Map>> orderEquivInfoMap) { assert sortKey instanceof RexInputRef : "sort key should be always RexInputRef"; @@ -238,17 +246,17 @@ public interface Config extends OpenSearchRuleConfig { .build() .withOperandSupplier( b0 -> - b0.operand(LogicalSort.class) + b0.operand(Sort.class) // Pure limit pushdown should be covered by SortProjectTransposeRule and // OpenSearchLimitIndexScanRule .predicate(sort -> !sort.collation.getFieldCollations().isEmpty()) .oneInput( b1 -> - b1.operand(LogicalProject.class) + b1.operand(Project.class) .predicate(Predicate.not(Project::containsOver)) .oneInput( b2 -> - b2.operand(CalciteLogicalIndexScan.class) + b2.operand(AbstractCalciteIndexScan.class) .predicate( AbstractCalciteIndexScan::noAggregatePushed) .noInputs()))); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortProjectExprTransposeRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortProjectExprTransposeRule.java index a40ca3877bc..9136b72a246 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortProjectExprTransposeRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/SortProjectExprTransposeRule.java @@ -20,8 +20,6 @@ import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Sort; -import org.apache.calcite.rel.logical.LogicalProject; -import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rex.RexNode; import org.apache.commons.lang3.tuple.Pair; import org.immutables.value.Value; @@ -48,6 +46,10 @@ protected void onMatchImpl(RelOptRuleCall call) { final Sort sort = call.rel(0); final Project project = call.rel(1); + if (sort.getConvention() != project.getConvention()) { + return; + } + List pushable = new ArrayList<>(); boolean allPushable = true; for (RelFieldCollation fieldCollation : sort.getCollation().getFieldCollations()) { @@ -128,12 +130,12 @@ public interface Config extends OpenSearchRuleConfig { .build() .withOperandSupplier( b0 -> - b0.operand(LogicalSort.class) + b0.operand(Sort.class) .oneInput( b1 -> - b1.operand(LogicalProject.class) + b1.operand(Project.class) .predicate( - Predicate.not(LogicalProject::containsOver) + Predicate.not(Project::containsOver) .and(PlanUtils::containsRexCall)) .anyInputs())); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 92d2940e842..65cba18a952 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -8,12 +8,15 @@ import static java.util.Objects.requireNonNull; import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_PUSHDOWN_ROWCOUNT_ESTIMATION_FACTOR; import static org.opensearch.sql.opensearch.storage.scan.context.PushDownType.AGGREGATION; +import static org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper.MISSING_MAX; import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.stream.Collectors; import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Stream; import lombok.Getter; import org.apache.calcite.adapter.enumerable.EnumerableMergeJoin; @@ -38,11 +41,15 @@ import org.apache.calcite.rel.metadata.RelMdUtil; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.util.NumberUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.checkerframework.checker.nullness.qual.Nullable; import org.opensearch.search.sort.ScoreSortBuilder; +import org.opensearch.search.sort.ScriptSortBuilder.ScriptSortType; import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortBuilders; import org.opensearch.search.sort.SortOrder; @@ -50,6 +57,8 @@ import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.opensearch.data.type.OpenSearchTextType; +import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder.PushDownUnSupportedException; +import org.opensearch.sql.opensearch.request.PredicateAnalyzer; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; import org.opensearch.sql.opensearch.storage.scan.context.AbstractAction; import org.opensearch.sql.opensearch.storage.scan.context.AggPushDownAction; @@ -276,6 +285,8 @@ public Map getAliasMapping() { return osIndex.getAliasMapping(); } + public abstract AbstractCalciteIndexScan copy(); + protected List getCollationNames(List collations) { return collations.stream() .map(collation -> getRowType().getFieldNames().get(collation.getFieldIndex())) @@ -435,6 +446,88 @@ && isAnyCollationNameInAggregators(collationNames)) { return null; } + /** + * Push down sort expressions to OpenSearch level. Supports mixed RexCall and field sort + * expressions. + * + * @param sortExprDigests List of SortExprDigest with expressions and collation information + * @return CalciteLogicalIndexScan with sort expressions pushed down, or null if pushdown fails + */ + public AbstractCalciteIndexScan pushdownSortExpr(List sortExprDigests) { + try { + if (sortExprDigests == null || sortExprDigests.isEmpty()) { + return null; + } + + AbstractCalciteIndexScan newScan = + buildScan( + getCluster(), + traitSet, + hints, + table, + osIndex, + getRowType(), + pushDownContext.cloneWithoutSort()); + + List>> sortBuilderSuppliers = new ArrayList<>(); + for (SortExprDigest digest : sortExprDigests) { + SortOrder order = + Direction.DESCENDING.equals(digest.getDirection()) ? SortOrder.DESC : SortOrder.ASC; + + if (digest.isSimpleFieldReference()) { + String missing = null; + switch (digest.getNullDirection()) { + case FIRST: + missing = "_first"; + break; + case LAST: + missing = "_last"; + break; + default: + break; + } + final String finalMissing = missing; + sortBuilderSuppliers.add( + () -> SortBuilders.fieldSort(digest.getFieldName()).order(order).missing(finalMissing)); + continue; + } + RexNode sortExpr = digest.getExpression(); + assert sortExpr instanceof RexCall : "sort expression should be RexCall"; + Map missingValueParams = + new LinkedHashMap<>() { + { + put(MISSING_MAX, digest.isMissingMax()); + } + }; + // Complex expression - use ScriptQueryExpression to generate script for sort + PredicateAnalyzer.ScriptQueryExpression scriptExpr = + new PredicateAnalyzer.ScriptQueryExpression( + digest.getExpression(), + rowType, + osIndex.getAllFieldTypes(), + getCluster(), + missingValueParams); + // Determine the correct ScriptSortType based on the expression's return type + ScriptSortType sortType = getScriptSortType(sortExpr.getType()); + + sortBuilderSuppliers.add( + () -> SortBuilders.scriptSort(scriptExpr.getScript(), sortType).order(order)); + } + + // Create action to push down sort expressions to OpenSearch + OSRequestBuilderAction action = + requestBuilder -> requestBuilder.pushDownSortSuppliers(sortBuilderSuppliers); + + newScan.pushDownContext.add(PushDownType.SORT_EXPR, sortExprDigests, action); + return newScan; + } catch (Exception e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Cannot pushdown sort expressions: {}", sortExprDigests, e); + } + } + return null; + } + /** * CalciteOpenSearchIndexScan doesn't allow push-down anymore (except Sort under some strict * condition) after Aggregate push-down. @@ -464,4 +557,22 @@ public boolean isScriptPushed() { public boolean isProjectPushed() { return this.getPushDownContext().isProjectPushed(); } + + /** + * Determine the appropriate ScriptSortType based on the expression's return type. + * + * @param relDataType the return type of the expression + * @return the appropriate ScriptSortType + */ + private ScriptSortType getScriptSortType(RelDataType relDataType) { + if (SqlTypeName.CHAR_TYPES.contains(relDataType.getSqlTypeName())) { + return ScriptSortType.STRING; + } else if (SqlTypeName.INT_TYPES.contains(relDataType.getSqlTypeName()) + || SqlTypeName.APPROX_TYPES.contains(relDataType.getSqlTypeName())) { + return ScriptSortType.NUMBER; + } else { + throw new PushDownUnSupportedException( + "Unsupported type for sort expression pushdown: " + relDataType); + } + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java index d2a8d4f43d9..af848b71fa1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java @@ -73,6 +73,12 @@ protected AbstractCalciteIndexScan buildScan( cluster, traitSet, hints, table, osIndex, schema, pushDownContext); } + @Override + public AbstractCalciteIndexScan copy() { + return new CalciteEnumerableIndexScan( + getCluster(), traitSet, hints, table, osIndex, schema, pushDownContext.clone()); + } + @Override public void register(RelOptPlanner planner) { for (RelOptRule rule : OpenSearchRules.OPEN_SEARCH_OPT_RULES) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index 5c3a0598579..2e97b64db57 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -5,15 +5,10 @@ package org.opensearch.sql.opensearch.storage.scan; -import static org.opensearch.sql.opensearch.storage.serde.ScriptParameterHelper.MISSING_MAX; - import com.google.common.collect.ImmutableList; -import java.util.ArrayList; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.Getter; import org.apache.calcite.plan.Convention; @@ -26,7 +21,6 @@ import org.apache.calcite.rel.RelCollation; import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelFieldCollation; -import org.apache.calcite.rel.RelFieldCollation.Direction; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Project; @@ -37,19 +31,13 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; -import org.opensearch.search.sort.ScriptSortBuilder.ScriptSortType; -import org.opensearch.search.sort.SortBuilder; -import org.opensearch.search.sort.SortBuilders; -import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.common.setting.Settings; @@ -60,7 +48,6 @@ import org.opensearch.sql.opensearch.planner.rules.EnumerableIndexScanRule; import org.opensearch.sql.opensearch.planner.rules.OpenSearchIndexRules; import org.opensearch.sql.opensearch.request.AggregateAnalyzer; -import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder.PushDownUnSupportedException; import org.opensearch.sql.opensearch.request.PredicateAnalyzer; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.QueryExpression; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; @@ -74,7 +61,6 @@ import org.opensearch.sql.opensearch.storage.scan.context.PushDownContext; import org.opensearch.sql.opensearch.storage.scan.context.PushDownType; import org.opensearch.sql.opensearch.storage.scan.context.RareTopDigest; -import org.opensearch.sql.opensearch.storage.scan.context.SortExprDigest; /** The logical relational operator representing a scan of an OpenSearchIndex type. */ @Getter @@ -117,6 +103,7 @@ protected AbstractCalciteIndexScan buildScan( cluster, traitSet, hints, table, osIndex, schema, pushDownContext); } + @Override public CalciteLogicalIndexScan copy() { return new CalciteLogicalIndexScan( getCluster(), traitSet, hints, table, osIndex, schema, pushDownContext.clone()); @@ -434,104 +421,4 @@ public AbstractRelNode pushDownLimit(LogicalSort sort, Integer limit, Integer of } return null; } - - /** - * Push down sort expressions to OpenSearch level. Supports mixed RexCall and field sort - * expressions. - * - * @param sortExprDigests List of SortExprDigest with expressions and collation information - * @return CalciteLogicalIndexScan with sort expressions pushed down, or null if pushdown fails - */ - public CalciteLogicalIndexScan pushdownSortExpr(List sortExprDigests) { - try { - if (sortExprDigests == null || sortExprDigests.isEmpty()) { - return null; - } - - CalciteLogicalIndexScan newScan = - new CalciteLogicalIndexScan( - getCluster(), - traitSet, - hints, - table, - osIndex, - getRowType(), - pushDownContext.cloneWithoutSort()); - - List>> sortBuilderSuppliers = new ArrayList<>(); - for (SortExprDigest digest : sortExprDigests) { - SortOrder order = - Direction.DESCENDING.equals(digest.getDirection()) ? SortOrder.DESC : SortOrder.ASC; - - if (digest.isSimpleFieldReference()) { - String missing = null; - switch (digest.getNullDirection()) { - case FIRST: - missing = "_first"; - break; - case LAST: - missing = "_last"; - break; - default: - break; - } - final String finalMissing = missing; - sortBuilderSuppliers.add( - () -> SortBuilders.fieldSort(digest.getFieldName()).order(order).missing(finalMissing)); - continue; - } - RexNode sortExpr = digest.getExpression(); - assert sortExpr instanceof RexCall : "sort expression should be RexCall"; - Map missingValueParams = - new LinkedHashMap<>() { - { - put(MISSING_MAX, digest.isMissingMax()); - } - }; - // Complex expression - use ScriptQueryExpression to generate script for sort - PredicateAnalyzer.ScriptQueryExpression scriptExpr = - new PredicateAnalyzer.ScriptQueryExpression( - digest.getExpression(), - rowType, - osIndex.getAllFieldTypes(), - getCluster(), - missingValueParams); - // Determine the correct ScriptSortType based on the expression's return type - ScriptSortType sortType = getScriptSortType(sortExpr.getType()); - - sortBuilderSuppliers.add( - () -> SortBuilders.scriptSort(scriptExpr.getScript(), sortType).order(order)); - } - - // Create action to push down sort expressions to OpenSearch - OSRequestBuilderAction action = - requestBuilder -> requestBuilder.pushDownSortSuppliers(sortBuilderSuppliers); - - newScan.pushDownContext.add(PushDownType.SORT_EXPR, sortExprDigests, action); - return newScan; - } catch (Exception e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Cannot pushdown sort expressions: {}", sortExprDigests, e); - } - } - return null; - } - - /** - * Determine the appropriate ScriptSortType based on the expression's return type. - * - * @param relDataType the return type of the expression - * @return the appropriate ScriptSortType - */ - private ScriptSortType getScriptSortType(RelDataType relDataType) { - if (SqlTypeName.CHAR_TYPES.contains(relDataType.getSqlTypeName())) { - return ScriptSortType.STRING; - } else if (SqlTypeName.INT_TYPES.contains(relDataType.getSqlTypeName()) - || SqlTypeName.APPROX_TYPES.contains(relDataType.getSqlTypeName())) { - return ScriptSortType.NUMBER; - } else { - throw new PushDownUnSupportedException( - "Unsupported type for sort expression pushdown: " + relDataType); - } - } } From ecac796a422755fd81063c87ea4a072c81e41cbb Mon Sep 17 00:00:00 2001 From: Songkan Tang Date: Thu, 11 Dec 2025 13:39:22 +0800 Subject: [PATCH 2/3] Fix known issue of IT failure in 2.19-dev Signed-off-by: Songkan Tang --- .../org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java index 986969d5085..1729374bd02 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java @@ -1022,7 +1022,7 @@ public void testComplexSortPushDownForSMJWithMaxOptionAndFieldList() throws IOEx executeQuery( String.format( "source=%s | eval name2=substring(name, 2, 1) | join max=1 name2,age [ source=%s |" - + " eval name2=substring(state, 2, 1) ]", + + " eval name2=substring(state, 2, 1) ] ｜ fields name, country, state, month, year, age, name2", TEST_INDEX_STATE_COUNTRY, TEST_INDEX_STATE_COUNTRY)); verifySchema( actual, From 90491dfc5b094f89312ed1b8fdd646f3264b02e2 Mon Sep 17 00:00:00 2001 From: Songkan Tang Date: Thu, 11 Dec 2025 14:24:11 +0800 Subject: [PATCH 3/3] Fix wrong pipe character Signed-off-by: Songkan Tang --- .../org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java index 1729374bd02..be91669b0d0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLJoinIT.java @@ -1022,7 +1022,7 @@ public void testComplexSortPushDownForSMJWithMaxOptionAndFieldList() throws IOEx executeQuery( String.format( "source=%s | eval name2=substring(name, 2, 1) | join max=1 name2,age [ source=%s |" - + " eval name2=substring(state, 2, 1) ] ｜ fields name, country, state, month, year, age, name2", + + " eval name2=substring(state, 2, 1) ] | fields name, country, state, month, year, age, name2", TEST_INDEX_STATE_COUNTRY, TEST_INDEX_STATE_COUNTRY)); verifySchema( actual,