diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/LogicalSystemLimit.java b/core/src/main/java/org/opensearch/sql/calcite/plan/LogicalSystemLimit.java index 6e46b63b976..84c448c8387 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/LogicalSystemLimit.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/LogicalSystemLimit.java @@ -39,17 +39,6 @@ public enum SystemLimitType { @Getter private final SystemLimitType type; - private LogicalSystemLimit( - SystemLimitType type, - RelOptCluster cluster, - RelTraitSet traitSet, - RelNode input, - RelCollation collation, - @Nullable RexNode offset, - @Nullable RexNode fetch) { - this(type, cluster, traitSet, Collections.emptyList(), input, collation, offset, fetch); - } - private LogicalSystemLimit( SystemLimitType type, RelOptCluster cluster, @@ -76,7 +65,8 @@ public static LogicalSystemLimit create( RelCollation collation = collations == null ? null : collations.get(0); collation = RelCollationTraitDef.INSTANCE.canonize(collation); RelTraitSet traitSet = input.getTraitSet().replace(Convention.NONE).replace(collation); - return new LogicalSystemLimit(type, cluster, traitSet, input, collation, offset, fetch); + return new LogicalSystemLimit( + type, cluster, traitSet, Collections.emptyList(), input, collation, offset, fetch); } @Override diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLHintStrategyTable.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLHintStrategyTable.java new file mode 100644 index 00000000000..84a8b437887 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLHintStrategyTable.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import com.google.common.base.Suppliers; +import java.util.function.Supplier; +import lombok.experimental.UtilityClass; +import org.apache.calcite.rel.hint.HintStrategyTable; +import org.apache.calcite.rel.logical.LogicalAggregate; + +@UtilityClass +public class PPLHintStrategyTable { + + private static final Supplier HINT_STRATEGY_TABLE = + Suppliers.memoize( + () -> + HintStrategyTable.builder() + .hintStrategy( + "stats_args", + (hint, rel) -> { + return rel instanceof LogicalAggregate; + }) + // add more here + .build()); + + /** Update the HINT_STRATEGY_TABLE when you create a new hint. */ + public static HintStrategyTable getHintStrategyTable() { + return HINT_STRATEGY_TABLE.get(); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java index 50e03fc608f..f00feed651e 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -29,7 +29,6 @@ import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Sort; import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.hint.HintStrategyTable; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalFilter; @@ -605,16 +604,7 @@ static void addIgnoreNullBucketHintToAggregate(RelBuilder relBuilder) { assert relBuilder.peek() instanceof LogicalAggregate : "Stats hits should be added to LogicalAggregate"; relBuilder.hints(statHits); - relBuilder - .getCluster() - .setHintStrategies( - HintStrategyTable.builder() - .hintStrategy( - "stats_args", - (hint, rel) -> { - return rel instanceof LogicalAggregate; - }) - .build()); + relBuilder.getCluster().setHintStrategies(PPLHintStrategyTable.getHintStrategyTable()); } /** Extract the RexLiteral from the aggregate call if the aggregate call is a LITERAL_AGG. */ diff --git a/core/src/main/java/org/opensearch/sql/exception/NonFallbackCalciteException.java b/core/src/main/java/org/opensearch/sql/exception/NonFallbackCalciteException.java index e11583178be..fae0af228dd 100644 --- a/core/src/main/java/org/opensearch/sql/exception/NonFallbackCalciteException.java +++ b/core/src/main/java/org/opensearch/sql/exception/NonFallbackCalciteException.java @@ -11,4 +11,8 @@ public class NonFallbackCalciteException extends QueryEngineException { public NonFallbackCalciteException(String message) { super(message); } + + public NonFallbackCalciteException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/docs/user/optimization/optimization.rst b/docs/user/optimization/optimization.rst index f60b51bddac..0af19e9f9ed 100644 --- a/docs/user/optimization/optimization.rst +++ b/docs/user/optimization/optimization.rst @@ -44,7 +44,7 @@ The consecutive Filter operator will be merged as one Filter operator:: { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"filter\":[{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":10,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}})" }, "children": [] } @@ -71,7 +71,7 @@ The Filter operator should be push down under Sort operator:: { "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, searchDone=false)" + "request": "OpenSearchQueryRequest(indexName=accounts, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":null,\"to\":20,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]})" }, "children": [] } @@ -102,7 +102,7 @@ The Project list will push down to Query DSL to `filter the source 10")); + expected = loadExpectedPlan("explain_agg_paginating_having2.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account | stats bucket_nullable = false count() by" + + " state | where `count()` > 10")); + expected = loadExpectedPlan("explain_agg_paginating_having3.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account | stats avg(balance) as avg, count() as cnt" + + " by state | eval new_avg = avg + 1000, new_cnt = cnt + 1 | where new_avg >" + + " 1000 or new_cnt > 1")); + } finally { + resetQueryBucketSize(); + } + } + + @Test + public void testPaginatingAggForJoin() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + try { + setQueryBucketSize(2); + String expected = loadExpectedPlan("explain_agg_paginating_join1.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account | stats count() as c by state | join left=l" + + " right=r on l.state=r.state [ source=opensearch-sql_test_index_bank | stats" + + " count() as c by state ]")); + expected = loadExpectedPlan("explain_agg_paginating_join2.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account | stats bucket_nullable = false count() as" + + " c by state | join left=l right=r on l.state=r.state [" + + " source=opensearch-sql_test_index_bank | stats bucket_nullable = false" + + " count() as c by state ]")); + expected = loadExpectedPlan("explain_agg_paginating_join3.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account | stats count() as c by state | join" + + " type=inner state [ source=opensearch-sql_test_index_bank | stats count()" + + " as c by state ]")); + expected = loadExpectedPlan("explain_agg_paginating_join4.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account | stats count() as c by state | head 10" + + " | join type=inner state [ source=opensearch-sql_test_index_account" + + " | stats count() as c by state ]")); + } finally { + resetQueryBucketSize(); + } + } + + @Test + public void testPaginatingAggForHeadFrom() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + try { + setQueryBucketSize(2); + String expected = loadExpectedPlan("explain_agg_paginating_head_from.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + "source=opensearch-sql_test_index_account | stats count() as c by state | head 10" + + " from 2")); + } finally { + resetQueryBucketSize(); + } + } + + @Test + public void testPaginatingHeadSizeNoLessThanQueryBucketSize() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + try { + setQueryBucketSize(2); + String expected = + loadExpectedPlan("explain_agg_paginating_head_size_query_bucket_size1.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | stats count() by age | sort -age | head 3", TEST_INDEX_BANK))); + expected = loadExpectedPlan("explain_agg_paginating_head_size_query_bucket_size2.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | stats count() by age | sort -age | head 2", TEST_INDEX_BANK))); + expected = loadExpectedPlan("explain_agg_paginating_head_size_query_bucket_size3.yaml"); + assertYamlEqualsIgnoreId( + expected, + explainQueryYaml( + String.format( + "source=%s | stats count() by age | sort -age | head 1", TEST_INDEX_BANK))); + } finally { + resetQueryBucketSize(); + } + } + @Test public void testExplainSortOnMeasure() throws IOException { enabledOnlyWhenPushdownIsEnabled(); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index e0872dc543c..9710d2f4415 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -1267,6 +1267,16 @@ public void testLimitAfterAggregation() throws IOException { "source=%s | stats count() by age | sort -age | head 3", TEST_INDEX_BANK)); verifySchema(response, schema("count()", "bigint"), schema("age", "int")); verifyDataRows(response, rows(1, 39), rows(2, 36), rows(1, 34)); + response = + executeQuery( + String.format( + "source=%s | stats count() by age | sort -age | head 2", TEST_INDEX_BANK)); + verifyDataRows(response, rows(1, 39), rows(2, 36)); + response = + executeQuery( + String.format( + "source=%s | stats count() by age | sort -age | head 1", TEST_INDEX_BANK)); + verifyDataRows(response, rows(1, 39)); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationPaginatingIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationPaginatingIT.java new file mode 100644 index 00000000000..4fef36ce76a --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationPaginatingIT.java @@ -0,0 +1,23 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import org.junit.After; + +public class CalcitePPLAggregationPaginatingIT extends CalcitePPLAggregationIT { + + @Override + public void init() throws Exception { + super.init(); + setQueryBucketSize(2); + } + + @After + public void tearDown() throws Exception { + resetQueryBucketSize(); + super.tearDown(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java index 19f95ec6c4e..8172bf81e62 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteStatsCommandIT.java @@ -5,6 +5,14 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.StatsCommandIT; public class CalciteStatsCommandIT extends StatsCommandIT { @@ -14,4 +22,88 @@ public void init() throws Exception { enableCalcite(); setQuerySizeLimit(2000); } + + @Test + public void testPaginatingStatsForHaving() throws IOException { + try { + setQueryBucketSize(2); + JSONObject response = + executeQuery( + String.format( + "source=%s | stats sum(balance) as a by state | where a > 780000", + TEST_INDEX_ACCOUNT)); + verifyDataRows(response, rows(782199, "TX")); + } finally { + resetQueryBucketSize(); + } + } + + @Test + public void testPaginatingStatsForJoin() throws IOException { + try { + setQueryBucketSize(2); + JSONObject response = + executeQuery( + String.format( + "source=%s | stats sum(balance) as a by state | join left=l right=r on l.state =" + + " r.state [ source = %s | stats sum(balance) as a by state ]", + TEST_INDEX_ACCOUNT, TEST_INDEX_BANK)); + verifyDataRows( + response, + rows(648774, "IL", 39225, "IL"), + rows(346934, "IN", 48086, "IN"), + rows(732523, "MD", 4180, "MD"), + rows(531785, "PA", 40540, "PA"), + rows(709135, "TN", 5686, "TN"), + rows(489601, "VA", 32838, "VA"), + rows(483741, "WA", 16418, "WA")); + } finally { + resetQueryBucketSize(); + } + } + + @Test + public void testPaginatingStatsForJoinField() throws IOException { + try { + setQueryBucketSize(2); + JSONObject response = + executeQuery( + String.format( + "source=%s | stats sum(balance) as a by state | join type=inner state " + + "[ source = %s | stats sum(balance) as a by state ]", + TEST_INDEX_ACCOUNT, TEST_INDEX_BANK)); + verifyDataRows( + response, + rows(39225, "IL"), + rows(48086, "IN"), + rows(4180, "MD"), + rows(40540, "PA"), + rows(5686, "TN"), + rows(32838, "VA"), + rows(16418, "WA")); + } finally { + resetQueryBucketSize(); + } + } + + @Test + public void testPaginatingStatsForHeadFrom() throws IOException { + try { + setQueryBucketSize(2); + JSONObject response = + executeQuery( + String.format( + "source=%s | stats sum(balance) as a by state | sort - a | head 5 from 2", + TEST_INDEX_ACCOUNT, TEST_INDEX_BANK)); + verifyDataRows( + response, + rows(710408, "MA"), + rows(709135, "TN"), + rows(657957, "ID"), + rows(648774, "IL"), + rows(643489, "AL")); + } finally { + resetQueryBucketSize(); + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLIntegTestCase.java index a607dc39f2b..29c84a947aa 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLIntegTestCase.java @@ -113,6 +113,7 @@ private Settings defaultSettings() { new ImmutableMap.Builder() .put(Key.QUERY_SIZE_LIMIT, 200) .put(Key.QUERY_BUCKET_SIZE, 1000) + .put(Key.SEARCH_MAX_BUCKETS, 65535) .put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1)) .put(Key.FIELD_TYPE_TOLERANCE, true) .put(Key.CALCITE_ENGINE_ENABLED, true) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchPaginatingIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchPaginatingIT.java new file mode 100644 index 00000000000..a9de5f732ae --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchPaginatingIT.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.tpch; + +import org.junit.After; +import org.opensearch.sql.util.Retry; + +@Retry +public class CalcitePPLTpchPaginatingIT extends CalcitePPLTpchIT { + + @Override + public void init() throws Exception { + super.init(); + setQueryBucketSize(2); + } + + @After + public void tearDown() throws Exception { + resetQueryBucketSize(); + super.tearDown(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java index 062880411ef..2d120ffc1e5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StandaloneIT.java @@ -167,6 +167,7 @@ private Settings defaultSettings() { new ImmutableMap.Builder() .put(Key.QUERY_SIZE_LIMIT, 200) .put(Key.QUERY_BUCKET_SIZE, 1000) + .put(Key.SEARCH_MAX_BUCKETS, 65535) .put(Key.SQL_CURSOR_KEEP_ALIVE, TimeValue.timeValueMinutes(1)) .put(Key.FIELD_TYPE_TOLERANCE, true) .build(); diff --git a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java index bf9b214bd44..079e332ef32 100644 --- a/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/util/MatcherUtils.java @@ -421,7 +421,7 @@ public static void assertJsonEqualsIgnoreId(String expected, String actual) { } private static String cleanUpId(String s) { - return eliminateTimeStamp(eliminatePid(eliminateRelId(s))); + return eliminateTimeStamp(eliminatePid(eliminateRelId(eliminateRequestOptions(s)))); } private static String eliminateTimeStamp(String s) { @@ -434,14 +434,17 @@ private static String eliminateRelId(String s) { .replaceAll("LogicalProject#\\d+", "LogicalProject#"); } + private static String eliminateRequestOptions(String s) { + return s.replaceAll(" needClean=true,", "").replaceAll(" searchDone=false,", ""); + } + private static String eliminatePid(String s) { return s.replaceAll("pitId=[^,]+,", "pitId=*,"); } /** Compare two YAML strings are equals with ignoring the RelNode id in the Calcite plan. */ public static void assertYamlEqualsIgnoreId(String expectedYaml, String actualYaml) { - String cleanedYaml = cleanUpYaml(actualYaml); - assertYamlEquals(expectedYaml, cleanedYaml); + assertYamlEquals(cleanUpYaml(expectedYaml), cleanUpYaml(actualYaml)); } public static void assertYamlEquals(String expected, String actual) { @@ -460,7 +463,9 @@ private static String cleanUpYaml(String s) { .replaceAll("rel#\\d+", "rel#") .replaceAll("RelSubset#\\d+", "RelSubset#") .replaceAll("LogicalProject#\\d+", "LogicalProject#") - .replaceAll("pitId=[^,]+,", "pitId=*,"); + .replaceAll("pitId=[^,]+,", "pitId=*,") + .replaceAll(" needClean=true,", "") + .replaceAll(" searchDone=false,", ""); } private static String jsonToYaml(String json) { diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_cannot_push.yaml index c9929d83e81..b7405bf3963 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_cannot_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_cannot_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, SEARCH($10, Sarg[[30..40]]), 'u40':VARCHAR, 'u100':VARCHAR)], age=[$10]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQGFHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNFQVJDSCIsCiAgICAgICAgImtpbmQiOiAiU0VBUkNIIiwKICAgICAgICAic3ludGF4IjogIklOVEVSTkFMIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDMsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgImNsb3NlZCIsCiAgICAgICAgICAgICAgICAiMzAiLAogICAgICAgICAgICAgICAgIjQwIgogICAgICAgICAgICAgIF0KICAgICAgICAgICAgXSwKICAgICAgICAgICAgIm51bGxBcyI6ICJVTktOT1dOIgogICAgICAgICAgfSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiA0LAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDUsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,0,2,2],"DIGESTS":["age",30,"u30","age","u40","u100"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg_age=AVG($1)), PROJECT->[avg_age, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQGFHsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIlNFQVJDSCIsCiAgICAgICAgImtpbmQiOiAiU0VBUkNIIiwKICAgICAgICAic3ludGF4IjogIklOVEVSTkFMIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDMsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogewogICAgICAgICAgICAicmFuZ2VTZXQiOiBbCiAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgImNsb3NlZCIsCiAgICAgICAgICAgICAgICAiMzAiLAogICAgICAgICAgICAgICAgIjQwIgogICAgICAgICAgICAgIF0KICAgICAgICAgICAgXSwKICAgICAgICAgICAgIm51bGxBcyI6ICJVTktOT1dOIgogICAgICAgICAgfSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiA0LAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDUsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,0,2,2],"DIGESTS":["age",30,"u30","age","u40","u100"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml index 20199016fb4..9ce326d7c7d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_composite_cannot_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, $11)], state=[$9], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDJXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDMsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,1],"DIGESTS":["age",35,"u35","email"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_balance=AVG($2)), PROJECT->[avg_balance, age_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQDJXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDMsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,1],"DIGESTS":["age",35,"u35","email"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_num_res_cannot_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_num_res_cannot_push.yaml index f50f5e44582..d4670d6dede 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_case_num_res_cannot_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_case_num_res_cannot_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(age_range=[CASE(<($10, 30), 30, 100)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC9HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,2],"DIGESTS":["age",30,30,100]}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age_range":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQC9HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiPCIsCiAgICAgICAgImtpbmQiOiAiTEVTU19USEFOIiwKICAgICAgICAic3ludGF4IjogIkJJTkFSWSIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAyLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2,2],"DIGESTS":["age",30,30,100]}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_count_push.yaml index 353bcf5c1e9..6d12ca7285e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_count_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], state=[$9], gender=[$4], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg(balance)=AVG($3),count()=COUNT()), PROJECT->[avg(balance), count(), age_range, state, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"a30","from":30.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg(balance)=AVG($3),count()=COUNT()), PROJECT->[avg(balance), count(), age_range, state, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"a30","from":30.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_range_count_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_range_count_push.yaml index eef2a7b23f8..4bbafd2928a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_range_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite2_range_range_count_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(age_range=[CASE(<($10, 35), 'u35':VARCHAR, 'a35':VARCHAR)], balance_range=[CASE(<($7, 20000), 'medium':VARCHAR, 'high':VARCHAR)], state=[$9], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg_balance=AVG($3)), PROJECT->[avg_balance, age_range, balance_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u35","to":35.0},{"key":"a35","from":35.0}],"keyed":true},"aggregations":{"balance_range":{"range":{"field":"balance","ranges":[{"key":"medium","to":20000.0},{"key":"high","from":20000.0}],"keyed":true},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg_balance=AVG($3)), PROJECT->[avg_balance, age_range, balance_range, state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u35","to":35.0},{"key":"a35","from":35.0}],"keyed":true},"aggregations":{"balance_range":{"range":{"field":"balance","ranges":[{"key":"medium","to":20000.0},{"key":"high","from":20000.0}],"keyed":true},"aggregations":{"avg_balance":{"avg":{"field":"balance"}}}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_range_metric_push.yaml index 14ee0aff97b..0c78841e2a1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_autodate_range_metric_push.yaml @@ -8,4 +8,4 @@ calcite: LogicalProject(@timestamp=[$0], category=[$1], value=[$2], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], timestamp=[WIDTH_BUCKET($3, 3, -(MAX($3) OVER (), MIN($3) OVER ()), MAX($3) OVER ())], value_range=[CASE(<($2, 7000), 'small':VARCHAR, 'great':VARCHAR)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg(value)=AVG($3),count()=COUNT()), PROJECT->[avg(value), count(), timestamp, value_range, category], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"timestamp":{"auto_date_histogram":{"field":"timestamp","buckets":3,"minimum_interval":null},"aggregations":{"value_range":{"range":{"field":"value","ranges":[{"key":"small","to":7000.0},{"key":"great","from":7000.0}],"keyed":true},"aggregations":{"avg(value)":{"avg":{"field":"value"}}}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},avg(value)=AVG($3),count()=COUNT()), PROJECT->[avg(value), count(), timestamp, value_range, category], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"category":{"terms":{"field":"category","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"timestamp":{"auto_date_histogram":{"field":"timestamp","buckets":3,"minimum_interval":null},"aggregations":{"value_range":{"range":{"field":"value","ranges":[{"key":"small","to":7000.0},{"key":"great","from":7000.0}],"keyed":true},"aggregations":{"avg(value)":{"avg":{"field":"value"}}}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_date_range_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_date_range_push.yaml index 30e4762d325..3c116fc7b44 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_date_range_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_date_range_push.yaml @@ -8,4 +8,4 @@ calcite: LogicalProject(@timestamp=[$0], category=[$1], value=[$2], timestamp=[$3], _id=[$4], _index=[$5], _score=[$6], _maxscore=[$7], _sort=[$8], _routing=[$9], value_range=[CASE(<($2, 7000), 'small':VARCHAR, 'large':VARCHAR)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(value)=AVG($1)), PROJECT->[avg(value), span(@timestamp,1h), value_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"@timestamp","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(@timestamp,1h)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1h"}}}]},"aggregations":{"value_range":{"range":{"field":"value","ranges":[{"key":"small","to":7000.0},{"key":"large","from":7000.0}],"keyed":true},"aggregations":{"avg(value)":{"avg":{"field":"value"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 2},avg(value)=AVG($1)), PROJECT->[avg(value), span(@timestamp,1h), value_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"@timestamp","boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(@timestamp,1h)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1h"}}}]},"aggregations":{"value_range":{"range":{"field":"value","ranges":[{"key":"small","to":7000.0},{"key":"large","from":7000.0}],"keyed":true},"aggregations":{"avg(value)":{"avg":{"field":"value"}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_metric_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_metric_push.yaml index 065598bc82c..605111dfc86 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_metric_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/agg_composite_range_metric_push.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(state=[$9], age_range=[CASE(<($10, 30), 'u30':VARCHAR, 'a30':VARCHAR)], balance=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[avg(balance), state, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"a30","from":30.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), PROJECT->[avg(balance), state, age_range], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"age_range":{"range":{"field":"age","ranges":[{"key":"u30","to":30.0},{"key":"a30","from":30.0}],"keyed":true},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml index 56dec15223d..6f0e2fdae76 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_date_histogram_daily.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2022-12-30 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-07 12:00:00':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2022-12-30 00:00:00':VARCHAR..'2023-01-07 12:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1d)], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2022-12-30T00:00:00.000Z","to":"2023-01-07T12:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"span(`@timestamp`,1d)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2022-12-30 00:00:00':VARCHAR..'2023-01-07 12:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1d)], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2022-12-30T00:00:00.000Z","to":"2023-01-07T12:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"span(`@timestamp`,1d)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml index 6d3ef26ee3e..9be838e68b8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-02 10:00:00':VARCHAR)), IS NOT NULL($7), IS NOT NULL($14))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp], FILTER->AND(SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, cloud.region], SORT->[1 DESC LAST, 2 ASC FIRST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"process.name","boost":1.0}},{"exists":{"field":"cloud.region","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["process.name","cloud.region","@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp], FILTER->AND(SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), IS NOT NULL($0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, cloud.region], SORT->[1 DESC LAST, 2 ASC FIRST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"process.name","boost":1.0}},{"exists":{"field":"cloud.region","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["process.name","cloud.region","@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml index 2b48025c015..9cab2b0de11 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/composite_terms_keyword.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-02 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-02 10:00:00':VARCHAR)), IS NOT NULL($7), IS NOT NULL($14), IS NOT NULL($34))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp, aws.cloudwatch.log_stream], FILTER->AND(SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), IS NOT NULL($0), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},count()=COUNT()), PROJECT->[count(), process.name, cloud.region, aws.cloudwatch.log_stream], SORT->[1 DESC LAST, 2 ASC FIRST, 3 ASC FIRST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"process.name","boost":1.0}},{"exists":{"field":"cloud.region","boost":1.0}},{"exists":{"field":"aws.cloudwatch.log_stream","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["process.name","cloud.region","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, cloud.region, @timestamp, aws.cloudwatch.log_stream], FILTER->AND(SEARCH($2, Sarg[['2023-01-02 00:00:00':VARCHAR..'2023-01-02 10:00:00':VARCHAR)]:VARCHAR), IS NOT NULL($0), IS NOT NULL($1), IS NOT NULL($3)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1, 2},count()=COUNT()), PROJECT->[count(), process.name, cloud.region, aws.cloudwatch.log_stream], SORT->[1 DESC LAST, 2 ASC FIRST, 3 ASC FIRST], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"range":{"@timestamp":{"from":"2023-01-02T00:00:00.000Z","to":"2023-01-02T10:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},{"exists":{"field":"process.name","boost":1.0}},{"exists":{"field":"cloud.region","boost":1.0}},{"exists":{"field":"aws.cloudwatch.log_stream","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["process.name","cloud.region","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":false,"order":"desc"}}},{"cloud.region":{"terms":{"field":"cloud.region","missing_bucket":false,"order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_hourly_agg.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_hourly_agg.yaml index a97ca073a21..2982377d2ca 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_hourly_agg.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_hourly_agg.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($17)]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1h)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1h)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1h"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1h)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1h)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1h"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml index 3d5639f94c0..7bccacdf5af 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/date_histogram_minute_agg.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1m)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[@timestamp], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(`@timestamp`,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["@timestamp"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"span(`@timestamp`,1m)":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_1.yaml index 2f3aab7b147..6e5b68f495c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_1.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), aws.cloudwatch.log_stream, process.name], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), aws.cloudwatch.log_stream, process.name], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_2.yaml index cf04d9b8695..5d9f9d1f579 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/big5/terms_significant_2.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(>=($17, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($17, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, big5]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, aws.cloudwatch.log_stream], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, big5]], PushDownContext=[[PROJECT->[process.name, @timestamp, aws.cloudwatch.log_stream], FILTER->SEARCH($1, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), process.name, aws.cloudwatch.log_stream], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"@timestamp":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["process.name","@timestamp","aws.cloudwatch.log_stream"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"process.name":{"terms":{"field":"process.name","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"aws.cloudwatch.log_stream":{"terms":{"field":"aws.cloudwatch.log_stream","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml index b011edc42f5..f8200b10a81 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_single_group_key.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($4), IS NOT NULL($7)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[FILTER->AND(IS NOT NULL($4), IS NOT NULL($7)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"gender","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml index c47fb9dd438..4fcc802063f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_integer_span.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(IS NOT NULL($10), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"age","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[balance, age], FILTER->AND(IS NOT NULL($1), IS NOT NULL($0)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},max(balance)=MAX($0)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"age","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["balance","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age0":{"histogram":{"field":"age","missing_bucket":false,"order":"asc","interval":10.0}}}]},"aggregations":{"max(balance)":{"max":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml index 389825459df..e4578f21fcd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_limit.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[AND(IS NOT NULL($9), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, state], FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"state","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["gender","balance","state"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[gender, balance, state], FILTER->AND(IS NOT NULL($2), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(balance)=AVG($2)), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"state","boost":1.0}},{"exists":{"field":"balance","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["gender","balance","state"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}},{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg(balance)":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml index a07c92033d0..8768b05a1a0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/chart_with_timestamp_span.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($0)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count=COUNT()), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_time_data]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count=COUNT()), SORT->[0], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"@timestamp0":{"date_histogram":{"field":"@timestamp","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q18.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q18.yaml index 59742c27ae9..7b3603d66aa 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q18.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q18.yaml @@ -8,4 +8,4 @@ calcite: LogicalFilter(condition=[AND(IS NOT NULL($84), IS NOT NULL($63))]) CalciteLogicalIndexScan(table=[[OpenSearch, hits]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), UserID, SearchPhrase], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"UserID":{"terms":{"field":"UserID","missing_bucket":false,"order":"asc"}}},{"SearchPhrase":{"terms":{"field":"SearchPhrase","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), UserID, SearchPhrase], LIMIT->10, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":10,"sources":[{"UserID":{"terms":{"field":"UserID","missing_bucket":false,"order":"asc"}}},{"SearchPhrase":{"terms":{"field":"SearchPhrase","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml index 2bf72da9393..fbc4dd965dd 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q28.yaml @@ -13,4 +13,4 @@ calcite: EnumerableLimit(fetch=[25]) EnumerableSort(sort0=[$0], dir0=[DESC-nulls-last]) EnumerableCalc(expr#0..2=[{inputs}], expr#3=[100000], expr#4=[>($t1, $t3)], proj#0..2=[{exprs}], $condition=[$t4]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[URL, CounterID], FILTER->AND(<>($0, ''), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($1),c=COUNT()), PROJECT->[l, c, CounterID]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"term":{"URL":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"CounterID","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["URL","CounterID"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"CounterID":{"terms":{"field":"CounterID","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["URL"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[URL, CounterID], FILTER->AND(<>($0, ''), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($1),c=COUNT()), PROJECT->[l, c, CounterID]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"bool":{"must":[{"exists":{"field":"URL","boost":1.0}}],"must_not":[{"term":{"URL":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},{"exists":{"field":"CounterID","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["URL","CounterID"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"CounterID":{"terms":{"field":"CounterID","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["URL"]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml index 97fdce0a1c0..08776c92111 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q29.yaml @@ -15,4 +15,4 @@ calcite: EnumerableLimit(fetch=[25]) EnumerableSort(sort0=[$0], dir0=[DESC-nulls-last]) EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100000], expr#5=[>($t1, $t4)], proj#0..3=[{exprs}], $condition=[$t5]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[Referer], FILTER-><>($0, ''), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($2),c=COUNT(),min(Referer)=MIN($1)), PROJECT->[l, c, min(Referer), k]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"Referer","boost":1.0}}],"must_not":[{"term":{"Referer":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["Referer"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"k":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCGXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IDIKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["Referer","^https?://(?:www\\.)?([^/]+)/.*$","$1"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["Referer"]}}}},"min(Referer)":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"Referer"}],"sort":[{"Referer":{"order":"asc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[Referer], FILTER-><>($0, ''), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},l=AVG($2),c=COUNT(),min(Referer)=MIN($1)), PROJECT->[l, c, min(Referer), k]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"exists":{"field":"Referer","boost":1.0}}],"must_not":[{"term":{"Referer":{"value":"","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["Referer"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":10000,"sources":[{"k":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCGXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IDIKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["Referer","^https?://(?:www\\.)?([^/]+)/.*$","$1"]}},"missing_bucket":false,"order":"asc"}}}]},"aggregations":{"l":{"avg":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["Referer"]}}}},"min(Referer)":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"Referer"}],"sort":[{"Referer":{"order":"asc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml index 820898e401d..d261f22d6f5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/clickbench/q43.yaml @@ -11,4 +11,4 @@ calcite: EnumerableCalc(expr#0..1=[{inputs}], PageViews=[$t1], M=[$t0]) EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[1000], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, EventTime, DontCountHits, IsRefresh, CounterID], FILTER->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-15 00:00:00':VARCHAR]]:VARCHAR), =($3, 0), =($2, 0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},PageViews=COUNT()), SORT->[0 ASC FIRST], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-15T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}},{"exists":{"field":"EventTime","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","EventTime","DontCountHits","IsRefresh","CounterID"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1010,"sources":[{"M":{"date_histogram":{"field":"EventTime","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, hits]], PushDownContext=[[PROJECT->[EventDate, EventTime, DontCountHits, IsRefresh, CounterID], FILTER->AND(=($4, 62), SEARCH($0, Sarg[['2013-07-01 00:00:00':VARCHAR..'2013-07-15 00:00:00':VARCHAR]]:VARCHAR), =($3, 0), =($2, 0), IS NOT NULL($1)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},PageViews=COUNT()), SORT->[0 ASC FIRST], LIMIT->[10 from 1000]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"bool":{"must":[{"term":{"CounterID":{"value":62,"boost":1.0}}},{"range":{"EventDate":{"from":"2013-07-01T00:00:00.000Z","to":"2013-07-15T00:00:00.000Z","include_lower":true,"include_upper":true,"format":"date_time","boost":1.0}}},{"term":{"IsRefresh":{"value":0,"boost":1.0}}},{"term":{"DontCountHits":{"value":0,"boost":1.0}}},{"exists":{"field":"EventTime","boost":1.0}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["EventDate","EventTime","DontCountHits","IsRefresh","CounterID"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1010,"sources":[{"M":{"date_histogram":{"field":"EventTime","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=1010, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml index 722be1bec9f..e731c2d3433 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], c1=[$t1], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml index 61724171f7a..026d83de0e8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], c1=[$t1], c2=[$t1], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c1=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"c1":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c1=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"c1":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml index a4dfdb25064..4e49ba34da0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], account_number=[$0], account_number_alias=[$0]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(account_number)=COUNT($1),c2=COUNT($2)), PROJECT->[count(account_number), c2, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(account_number)":{"value_count":{"field":"account_number"}},"c2":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(account_number)=COUNT($1),c2=COUNT($2)), PROJECT->[count(account_number), c2, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(account_number)":{"value_count":{"field":"account_number"}},"c2":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml index 77fc6c6eadf..039d6668fd8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], account_number=[$0]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT(),count(account_number)=COUNT($1)), PROJECT->[count(), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT(),count(account_number)=COUNT($1)), PROJECT->[count(), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml index dc7bf3629f2..40e1333630a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], balance=[$3], account_number=[$0]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(balance)=COUNT($1),count(account_number)=COUNT($2)), PROJECT->[count(balance), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(balance)":{"value_count":{"field":"balance"}},"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(balance)=COUNT($1),count(account_number)=COUNT($2)), PROJECT->[count(balance), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(balance)":{"value_count":{"field":"balance"}},"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml index b0b23b7ce10..138cdec9369 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], b_1=[+($3, 1)], $f3=[POWER($3, 2)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBVXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",2]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",1]}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBVXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",2]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml index aaf4f2017cb..acd95f0ec63 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_group_merge.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[*($t0, $t2)], expr#4=[+($t0, $t2)], count()=[$t1], age1=[$t3], age2=[$t4], age3=[$t2], age=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having1.yaml new file mode 100644 index 00000000000..bc06e988e1f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having1.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalFilter(condition=[>($0, 10)]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[>($t0, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having2.yaml new file mode 100644 index 00000000000..8f1a667a604 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having2.yaml @@ -0,0 +1,13 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalFilter(condition=[>($0, 10)]) + LogicalProject(count()=[$1], state=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(state=[$7]) + LogicalFilter(condition=[IS NOT NULL($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[10], expr#3=[>($t0, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml new file mode 100644 index 00000000000..e7589d8109d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_having3.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalFilter(condition=[OR(>($3, 1000), >($4, 1))]) + LogicalProject(avg=[$1], cnt=[$2], state=[$0], new_avg=[+($1, 1000)], new_cnt=[+($2, 1)]) + LogicalAggregate(group=[{0}], avg=[AVG($1)], cnt=[COUNT()]) + LogicalProject(state=[$7], balance=[$3]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1000], expr#4=[+($t1, $t3)], expr#5=[1], expr#6=[+($t2, $t5)], expr#7=[>($t4, $t3)], expr#8=[>($t6, $t5)], expr#9=[OR($t7, $t8)], avg=[$t1], cnt=[$t2], state=[$t0], new_avg=[$t4], new_cnt=[$t6], $condition=[$t9]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg=AVG($1),cnt=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg":{"avg":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_from.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_from.yaml new file mode 100644 index 00000000000..cfe1a0a29df --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_from.yaml @@ -0,0 +1,12 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(offset=[2], fetch=[10]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableLimit(offset=[2], fetch=[10]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_size_query_bucket_size1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_size_query_bucket_size1.yaml new file mode 100644 index 00000000000..5663ba56f9b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_size_query_bucket_size1.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[DESC-nulls-last], fetch=[3]) + LogicalProject(count()=[$1], age=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), age], SORT->[1 DESC LAST], LIMIT->3, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"last","order":"desc"}}}]}}}}, requestedTotalSize=3, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_size_query_bucket_size2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_size_query_bucket_size2.yaml new file mode 100644 index 00000000000..c60fc184061 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_size_query_bucket_size2.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[DESC-nulls-last], fetch=[2]) + LogicalProject(count()=[$1], age=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), age], SORT->[1 DESC LAST], LIMIT->2, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"last","order":"desc"}}}]}}}}, requestedTotalSize=2, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_size_query_bucket_size3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_size_query_bucket_size3.yaml new file mode 100644 index 00000000000..835a72a6f55 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_head_size_query_bucket_size3.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(sort0=[$1], dir0=[DESC-nulls-last], fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalSort(sort0=[$1], dir0=[DESC-nulls-last], fetch=[1]) + LogicalProject(count()=[$1], age=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(age=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), age], SORT->[1 DESC LAST], LIMIT->1, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1,"sources":[{"age":{"terms":{"field":"age","missing_bucket":true,"missing_order":"last","order":"desc"}}}]}}}}, requestedTotalSize=1, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join1.yaml new file mode 100644 index 00000000000..ea76cdee61e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join1.yaml @@ -0,0 +1,19 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c=[$0], state=[$1], r.c=[$2], r.state=[$3]) + LogicalJoin(condition=[=($1, $3)], joinType=[inner]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$9]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[inner]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], LIMIT->50000, SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join2.yaml new file mode 100644 index 00000000000..ca7ab3fc668 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join2.yaml @@ -0,0 +1,21 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c=[$0], state=[$1], r.c=[$2], r.state=[$3]) + LogicalJoin(condition=[=($1, $3)], joinType=[inner]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$7]) + LogicalFilter(condition=[IS NOT NULL($7)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$9]) + LogicalFilter(condition=[IS NOT NULL($9)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableMergeJoin(condition=[=($1, $3)], joinType=[inner]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], LIMIT->50000, SORT->[1]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join3.yaml new file mode 100644 index 00000000000..1326030ea7e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join3.yaml @@ -0,0 +1,19 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c=[$2], state=[$3]) + LogicalJoin(condition=[=($1, $3)], joinType=[inner]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$9]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableHashJoin(condition=[=($1, $2)], joinType=[semi]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], LIMIT->50000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0}), SORT->[0]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join4.yaml new file mode 100644 index 00000000000..51d82f37450 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_paginating_join4.yaml @@ -0,0 +1,20 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c=[$2], state=[$3]) + LogicalJoin(condition=[=($1, $3)], joinType=[inner]) + LogicalSort(fetch=[10]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + LogicalSystemLimit(fetch=[50000], type=[JOIN_SUBSEARCH_MAXOUT]) + LogicalProject(c=[$1], state=[$0]) + LogicalAggregate(group=[{0}], c=[COUNT()]) + LogicalProject(state=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableHashJoin(condition=[=($1, $2)], joinType=[semi]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c=COUNT()), PROJECT->[c, state], LIMIT->50000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=50000, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0}), LIMIT->10], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":2,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"last","order":"asc"}}}]}}}}, requestedTotalSize=10, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.yaml index 03ae462f020..ad53f3b5972 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.yaml @@ -7,4 +7,4 @@ calcite: LogicalProject(t=[UNIX_TIMESTAMP($3)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), t], SORT->[1 ASC FIRST], LIMIT->3, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":3,"sources":[{"t":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQB4XsKICAib3AiOiB7CiAgICAibmFtZSI6ICJVTklYX1RJTUVTVEFNUCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["birthdate"]}},"missing_bucket":true,"value_type":"double","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), t], SORT->[1 ASC FIRST], LIMIT->3, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":3,"sources":[{"t":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQB4XsKICAib3AiOiB7CiAgICAibmFtZSI6ICJVTklYX1RJTUVTVEFNUCIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiRE9VQkxFIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0],"DIGESTS":["birthdate"]}},"missing_bucket":true,"value_type":"double","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=3, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml index 483b5d2dfa0..fb0beac704b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.yaml @@ -8,4 +8,4 @@ calcite: LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], t=[DATE_ADD($3, 1:INTERVAL DAY)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(t,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(t,1d)":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQFmXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJTUEFOIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJEQVRFX0FERCIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAxCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ1ZHQiOiAiRVhQUl9USU1FU1RBTVAiLAogICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IC0xCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["birthdate",1,"d"]}},"missing_bucket":false,"value_type":"long","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(t,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(t,1d)":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQFmXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJTUEFOIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJEQVRFX0FERCIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAxCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ1ZHQiOiAiRVhQUl9USU1FU1RBTVAiLAogICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAibnVsbGFibGUiOiB0cnVlLAogICAgInByZWNpc2lvbiI6IC0xCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,2],"DIGESTS":["birthdate",1,"d"]}},"missing_bucket":false,"value_type":"long","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_distinct_count_approx_enhancement.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_distinct_count_approx_enhancement.json index a8d07145562..ceeaa784c82 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_distinct_count_approx_enhancement.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_distinct_count_approx_enhancement.json @@ -1,6 +1,6 @@ { "calcite":{ "logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(distinct_states=[$1], gender=[$0])\n LogicalAggregate(group=[{0}], distinct_states=[DISTINCT_COUNT_APPROX($1)])\n LogicalProject(gender=[$4], state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical":"CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},distinct_states=DISTINCT_COUNT_APPROX($1)), PROJECT->[distinct_states, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"distinct_states\":{\"cardinality\":{\"field\":\"state.keyword\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical":"CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},distinct_states=DISTINCT_COUNT_APPROX($1)), PROJECT->[distinct_states, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"distinct_states\":{\"cardinality\":{\"field\":\"state.keyword\"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml index 0726597ef90..265e25dd417 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CHAR_LENGTH($t0)], sum=[$t1], len=[$t2], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum=SUM($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum=SUM($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQBRXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAgICJudWxsYWJsZSI6IHRydWUKICAgICAgfQogICAgfSwKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml index 8fd23589e33..88b1e828eb6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"sum(balance)":{"sum":{"field":"balance"}},"sum(balance + 100)_COUNT":{"value_count":{"field":"balance"}},"sum(balance / 100)":{"sum":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCEXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICB9CiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2],"DIGESTS":["balance",100]}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.json deleted file mode 100644 index 7a857107a13..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalUnion(all=[true])\n LogicalProject(cnt=[$1], gender=[$0])\n LogicalAggregate(group=[{0}], cnt=[COUNT($1)])\n LogicalProject(gender=[$4], balance=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(cnt=[$0], gender=[null:VARCHAR])\n LogicalAggregate(group=[{}], cnt=[COUNT()])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableUnion(all=[true])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT($1)), PROJECT->[cnt, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[null:VARCHAR], proj#0..1=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"track_total_hits\":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.yaml new file mode 100644 index 00000000000..6180edc83f3 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.yaml @@ -0,0 +1,17 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalUnion(all=[true]) + LogicalProject(cnt=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], cnt=[COUNT($1)]) + LogicalProject(gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(cnt=[$0], gender=[null:VARCHAR]) + LogicalAggregate(group=[{}], cnt=[COUNT()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableUnion(all=[true]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT($1)), PROJECT->[cnt, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"cnt":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[null:VARCHAR], proj#0..1=[{exprs}]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_appendpipe_command.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_appendpipe_command.json index 6ec42972a10..1375ff21c2b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_appendpipe_command.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_appendpipe_command.json @@ -1,6 +1,6 @@ { "calcite": { "logical":"LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], cnt=[$19])\n LogicalUnion(all=[true])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], cnt=[null:BIGINT])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(account_number=[null:BIGINT], firstname=[null:VARCHAR], address=[null:VARCHAR], birthdate=[null:EXPR_TIMESTAMP VARCHAR], gender=[$0], city=[null:VARCHAR], lastname=[null:VARCHAR], balance=[null:BIGINT], employer=[null:VARCHAR], state=[null:VARCHAR], age=[null:INTEGER], email=[null:VARCHAR], male=[null:BOOLEAN], _id=[null:VARCHAR], _index=[null:VARCHAR], _score=[null:REAL], _maxscore=[null:REAL], _sort=[null:BIGINT], _routing=[null:VARCHAR], cnt=[$1])\n LogicalAggregate(group=[{0}], cnt=[COUNT($1)])\n LogicalProject(gender=[$4], balance=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical":"EnumerableLimit(fetch=[10000])\n EnumerableUnion(all=[true])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[null:BIGINT], proj#0..13=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[null:BIGINT], expr#3=[null:VARCHAR], expr#4=[null:EXPR_TIMESTAMP VARCHAR], expr#5=[null:INTEGER], expr#6=[null:BOOLEAN], account_number=[$t2], firstname=[$t3], address=[$t3], birthdate=[$t4], gender=[$t0], city=[$t3], lastname=[$t3], balance=[$t2], employer=[$t3], state=[$t3], age=[$t5], email=[$t3], male=[$t6], cnt=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical":"EnumerableLimit(fetch=[10000])\n EnumerableUnion(all=[true])\n EnumerableCalc(expr#0..12=[{inputs}], expr#13=[null:BIGINT], proj#0..13=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[account_number, firstname, address, birthdate, gender, city, lastname, balance, employer, state, age, email, male], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"account_number\",\"firstname\",\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\",\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[null:BIGINT], expr#3=[null:VARCHAR], expr#4=[null:EXPR_TIMESTAMP VARCHAR], expr#5=[null:INTEGER], expr#6=[null:BOOLEAN], account_number=[$t2], firstname=[$t3], address=[$t3], birthdate=[$t4], gender=[$t0], city=[$t3], lastname=[$t3], balance=[$t2], employer=[$t3], state=[$t3], age=[$t5], email=[$t3], male=[$t6], cnt=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex1.yaml index 1b4c379f9b6..61017e4f76b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex1.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($4)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","account_number","firstname","address","balance","city","employer","state","age","email","lastname"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","account_number","firstname","address","balance","city","employer","state","age","email","lastname"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex2.yaml index 6d5c76b1443..daca12143df 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex2.yaml @@ -8,4 +8,4 @@ calcite: LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age, state], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","state","account_number","age"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age, state], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","state","account_number","age"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex3.yaml index af72f304286..b5d9a2194dc 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex3.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex3.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","state","account_number","firstname","address","balance","city","employer","age","email","lastname"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","state","account_number","firstname","address","balance","city","employer","age","email","lastname"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex4.yaml index a2385ccdbb2..3dc325d896b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_complex4.yaml @@ -8,4 +8,4 @@ calcite: LogicalProject(account_number=[$0], gender=[$4], age=[$8], state=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age, state], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","state","account_number","age"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age, state], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0, 1},agg#0=LITERAL_AGG(2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","gender","age","state"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}},{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f2":{"top_hits":{"from":0,"size":2,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","state","account_number","age"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_false_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_false_push.yaml index c16dde54e22..4084a34e2d4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_false_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_keepempty_false_push.yaml @@ -8,4 +8,4 @@ calcite: LogicalProject(account_number=[$0], gender=[$4], age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","gender","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","account_number","age"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","gender","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","account_number","age"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_push.yaml index c16dde54e22..4084a34e2d4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_dedup_push.yaml @@ -8,4 +8,4 @@ calcite: LogicalProject(account_number=[$0], gender=[$4], age=[$8]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","gender","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","account_number","age"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, gender, age], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=LogicalProject#,group={0},agg#0=LITERAL_AGG(1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","_source":{"includes":["account_number","gender","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"$f1":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"_source":{"includes":["gender","account_number","age"],"excludes":[]},"script_fields":{}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml index 3dbd4ff8c20..3938446b735 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(server=[$1], message=[$3], @timestamp=[$2]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, server], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"server":{"terms":{"field":"server","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"earliest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"@timestamp":{"order":"asc"}}]}},"latest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"@timestamp":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, server], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"server":{"terms":{"field":"server","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"earliest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"@timestamp":{"order":"asc"}}]}},"latest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"@timestamp":{"order":"desc"}}]}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml index 4dc24423d9d..d38a91f0922 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(level=[$4], message=[$3], created_at=[$0]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, level], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"level":{"terms":{"field":"level","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"earliest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"created_at":{"order":"asc"}}]}},"latest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"created_at":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, level], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"level":{"terms":{"field":"level","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"earliest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"created_at":{"order":"asc"}}]}},"latest_message":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"message.keyword"}],"sort":[{"created_at":{"order":"desc"}}]}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.json deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml index c4f023585c2..f1b2c6a434f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[>($8, 30)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state, city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["city","state","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state, city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["city","state","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml index dd1f6444d09..9ac2557ab2e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_with_search.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[AND(>=($3, TIMESTAMP('2023-01-01 00:00:00':VARCHAR)), <($3, TIMESTAMP('2023-01-03 00:00:00':VARCHAR)))]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"birthdate":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["birthdate"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->SEARCH($0, Sarg[['2023-01-01 00:00:00':VARCHAR..'2023-01-03 00:00:00':VARCHAR)]:VARCHAR), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"birthdate":{"from":"2023-01-01T00:00:00.000Z","to":"2023-01-03T00:00:00.000Z","include_lower":true,"include_upper":false,"format":"date_time","boost":1.0}}},"_source":{"includes":["birthdate"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1d)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1d"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml index 30f505a24a7..e339a485677 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(gender=[$4], firstname=[$1]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},first_name=FIRST($1),last_name=LAST($1)), PROJECT->[first_name, last_name, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"first_name":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname"}]}},"last_name":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname"}],"sort":[{"_doc":{"order":"desc"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},first_name=FIRST($1),last_name=LAST($1)), PROJECT->[first_name, last_name, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"first_name":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname"}]}},"last_name":{"top_hits":{"from":0,"size":1,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"firstname"}],"sort":[{"_doc":{"order":"desc"}}]}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json index 5fbc3abecf1..82c72695d0b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.yaml index 3a19f39e7da..9a3b2cfe48c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.yaml @@ -7,4 +7,4 @@ calcite: LogicalProject(state=[$7]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->100, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":100,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->100, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":100,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=100, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json index 986459fd2f4..0f391511f22 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=20, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.yaml index d13ed6aa8d3..4df04144f06 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.yaml @@ -10,4 +10,4 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[10], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], SORT->[1 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], SORT->[1 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=20, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.yaml index b4117a5a84c..4ff9804f458 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.yaml @@ -12,4 +12,4 @@ calcite: EnumerableLimit(fetch=[10000]) EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0]) EnumerableLimit(offset=[10], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=20, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.yaml index 46e8cab736b..e4f66cf442d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.yaml @@ -11,4 +11,4 @@ calcite: physical: | EnumerableLimit(fetch=[10000]) EnumerableLimit(offset=[10], fetch=[10]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], SORT->[1 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], SORT->[1 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":20,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":false,"order":"asc"}}}]}}}}, requestedTotalSize=20, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml index 195971e53bc..30ad4020d5d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.yaml @@ -18,4 +18,4 @@ calcite: EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3]) EnumerableWindow(window#0=[window(partition {0} order by [0] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])]) EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], age2=[$t2], $condition=[$t3]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), SORT->[0 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["city","state","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), SORT->[0 ASC FIRST], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"range":{"age":{"from":30,"to":null,"include_lower":false,"include_upper":true,"boost":1.0}}},"_source":{"includes":["city","state","age"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"state":{"terms":{"field":"state.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}},{"city":{"terms":{"field":"city.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"avg_age":{"avg":{"field":"age"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml index dfcb6e6b7e0..cc07aa4bfc4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.yaml @@ -7,4 +7,4 @@ calcite: CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t1], tokens=[$t9], sample_logs=[$t2]) - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"patterns_field":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQG6HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0VBUkNIIiwKICAgICAgICAia2luZCI6ICJTRUFSQ0giLAogICAgICAgICJzeW50YXgiOiAiSU5URVJOQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IHsKICAgICAgICAgICAgInJhbmdlU2V0IjogWwogICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICJzaW5nbGV0b24iLAogICAgICAgICAgICAgICAgIntcInZhbHVlXCI6XCJcIixcImNoYXJzZXROYW1lXCI6XCJJU08tODg1OS0xXCIsXCJjb2xsYXRpb25cIjp7XCJjb2xsYXRpb25OYW1lXCI6XCJJU08tODg1OS0xJGVuX1VTJHByaW1hcnlcIixcImNvZXJjaWJpbGl0eVwiOlwiSU1QTElDSVRcIixcImxvY2FsZVwiOlwiZW5fVVNcIn0sXCJjaGFyc2V0XCI6XCJJU08tODg1OS0xXCIsXCJ2YWx1ZUJ5dGVzXCI6bnVsbH0iCiAgICAgICAgICAgICAgXQogICAgICAgICAgICBdLAogICAgICAgICAgICAibnVsbEFzIjogIlRSVUUiCiAgICAgICAgICB9LAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDQsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0,2,2],"DIGESTS":["email.keyword","","email.keyword","[a-zA-Z0-9]+","<*>"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"pattern_count":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQG6HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0VBUkNIIiwKICAgICAgICAia2luZCI6ICJTRUFSQ0giLAogICAgICAgICJzeW50YXgiOiAiSU5URVJOQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IHsKICAgICAgICAgICAgInJhbmdlU2V0IjogWwogICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICJzaW5nbGV0b24iLAogICAgICAgICAgICAgICAgIntcInZhbHVlXCI6XCJcIixcImNoYXJzZXROYW1lXCI6XCJJU08tODg1OS0xXCIsXCJjb2xsYXRpb25cIjp7XCJjb2xsYXRpb25OYW1lXCI6XCJJU08tODg1OS0xJGVuX1VTJHByaW1hcnlcIixcImNvZXJjaWJpbGl0eVwiOlwiSU1QTElDSVRcIixcImxvY2FsZVwiOlwiZW5fVVNcIn0sXCJjaGFyc2V0XCI6XCJJU08tODg1OS0xXCIsXCJ2YWx1ZUJ5dGVzXCI6bnVsbH0iCiAgICAgICAgICAgICAgXQogICAgICAgICAgICBdLAogICAgICAgICAgICAibnVsbEFzIjogIlRSVUUiCiAgICAgICAgICB9LAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDQsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0,2,2],"DIGESTS":["email.keyword","","email.keyword","[a-zA-Z0-9]+","<*>"]}}}},"sample_logs":{"top_hits":{"from":0,"size":10,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"email.keyword"}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"patterns_field":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQG6HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0VBUkNIIiwKICAgICAgICAia2luZCI6ICJTRUFSQ0giLAogICAgICAgICJzeW50YXgiOiAiSU5URVJOQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IHsKICAgICAgICAgICAgInJhbmdlU2V0IjogWwogICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICJzaW5nbGV0b24iLAogICAgICAgICAgICAgICAgIntcInZhbHVlXCI6XCJcIixcImNoYXJzZXROYW1lXCI6XCJJU08tODg1OS0xXCIsXCJjb2xsYXRpb25cIjp7XCJjb2xsYXRpb25OYW1lXCI6XCJJU08tODg1OS0xJGVuX1VTJHByaW1hcnlcIixcImNvZXJjaWJpbGl0eVwiOlwiSU1QTElDSVRcIixcImxvY2FsZVwiOlwiZW5fVVNcIn0sXCJjaGFyc2V0XCI6XCJJU08tODg1OS0xXCIsXCJ2YWx1ZUJ5dGVzXCI6bnVsbH0iCiAgICAgICAgICAgICAgXQogICAgICAgICAgICBdLAogICAgICAgICAgICAibnVsbEFzIjogIlRSVUUiCiAgICAgICAgICB9LAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDQsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0,2,2],"DIGESTS":["email.keyword","","email.keyword","[a-zA-Z0-9]+","<*>"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"pattern_count":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQG6HsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDQVNFIiwKICAgICJraW5kIjogIkNBU0UiLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiU0VBUkNIIiwKICAgICAgICAia2luZCI6ICJTRUFSQ0giLAogICAgICAgICJzeW50YXgiOiAiSU5URVJOQUwiCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAibGl0ZXJhbCI6IHsKICAgICAgICAgICAgInJhbmdlU2V0IjogWwogICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICJzaW5nbGV0b24iLAogICAgICAgICAgICAgICAgIntcInZhbHVlXCI6XCJcIixcImNoYXJzZXROYW1lXCI6XCJJU08tODg1OS0xXCIsXCJjb2xsYXRpb25cIjp7XCJjb2xsYXRpb25OYW1lXCI6XCJJU08tODg1OS0xJGVuX1VTJHByaW1hcnlcIixcImNvZXJjaWJpbGl0eVwiOlwiSU1QTElDSVRcIixcImxvY2FsZVwiOlwiZW5fVVNcIn0sXCJjaGFyc2V0XCI6XCJJU08tODg1OS0xXCIsXCJ2YWx1ZUJ5dGVzXCI6bnVsbH0iCiAgICAgICAgICAgICAgXQogICAgICAgICAgICBdLAogICAgICAgICAgICAibnVsbEFzIjogIlRSVUUiCiAgICAgICAgICB9LAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMSwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAgICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgICB9CiAgICAgICAgfSwKICAgICAgICB7CiAgICAgICAgICAiZHluYW1pY1BhcmFtIjogMywKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICAgICAgfQogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDQsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0KICBdCn0=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[0,2,0,2,2],"DIGESTS":["email.keyword","","email.keyword","[a-zA-Z0-9]+","<*>"]}}}},"sample_logs":{"top_hits":{"from":0,"size":10,"version":false,"seq_no_primary_term":false,"explain":false,"fields":[{"field":"email.keyword"}]}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.yaml index a5f43d257ce..90e144cad8f 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[>(CHAR_LENGTH($2), 0)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address], SCRIPT->>(CHAR_LENGTH($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), address_length], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCIXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI+IiwKICAgICJraW5kIjogIkdSRUFURVJfVEhBTiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address",0]}},"boost":1.0}},"_source":{"includes":["address"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"address_length":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["address"]}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address], SCRIPT->>(CHAR_LENGTH($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), address_length], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"script":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQCIXsKICAib3AiOiB7CiAgICAibmFtZSI6ICI+IiwKICAgICJraW5kIjogIkdSRUFURVJfVEhBTiIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgIm9wIjogewogICAgICAgICJuYW1lIjogIkNIQVJfTEVOR1RIIiwKICAgICAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAwLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0KICAgIH0sCiAgICB7CiAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1,2],"DIGESTS":["address",0]}},"boost":1.0}},"_source":{"includes":["address"],"excludes":[]},"aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"address_length":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQA/3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[1],"DIGESTS":["address"]}},"missing_bucket":true,"value_type":"long","missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json index 5b355b16b0e..75fea09dc13 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg(balance)=[$1], state=[$0])\n LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)])\n LogicalProject(state=[$7], balance=[$3])\n LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, state, age], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), PROJECT->[avg(balance), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\",\"state\",\"age\"],\"excludes\":[]},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(balance)\":{\"avg\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, state, age], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), PROJECT->[avg(balance), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\",\"state\",\"age\"],\"excludes\":[]},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(balance)\":{\"avg\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml index 14cf8e6db82..b4577ff19a4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term.yaml @@ -8,4 +8,4 @@ calcite: LogicalProject(environment=[$0], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"region":{"terms":{"field":"region","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"region":{"terms":{"field":"region","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml index 09e016d8e9e..02f72c2be02 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time_and_term2.yaml @@ -8,4 +8,4 @@ calcite: LogicalProject(environment=[$0], status_code=[$2], service=[$3], host=[$4], memory_usage=[$5], response_time=[$6], cpu_usage=[$7], region=[$8], bytes_sent=[$9], _id=[$10], _index=[$11], _score=[$12], _maxscore=[$13], _sort=[$14], _routing=[$15], @timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())]) CalciteLogicalIndexScan(table=[[OpenSearch, events]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(cpu_usage)=AVG($2)), PROJECT->[avg(cpu_usage), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"region":{"terms":{"field":"region","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg(cpu_usage)=AVG($2)), PROJECT->[avg(cpu_usage), @timestamp, region], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"region":{"terms":{"field":"region","missing_bucket":false,"order":"asc"}}}]},"aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json index 01258e3fd0b..094516a934a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"interval\":10.0}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"interval\":10.0}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json index 977fe96090c..6104070d6d4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n LogicalFilter(condition=[IS NOT NULL($10)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":false,\"order\":\"asc\",\"interval\":10.0}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":false,\"order\":\"asc\",\"interval\":10.0}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.yaml index b4384528c0c..0e1e29e9370 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($3)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1m)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1m)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","fixed_interval":"1m"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.yaml index 5021adf62b8..c9e64c16e80 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.yaml @@ -7,4 +7,4 @@ calcite: LogicalFilter(condition=[IS NOT NULL($3)]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1M)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1M)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","calendar_interval":"1M"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1M)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"span(birthdate,1M)":{"date_histogram":{"field":"birthdate","missing_bucket":false,"order":"asc","calendar_interval":"1M"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml b/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml index 71a1e22775c..0b6874a6499 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml +++ b/integ-test/src/test/resources/expectedOutput/calcite/udf_geoip_in_agg_pushed.yaml @@ -6,4 +6,4 @@ calcite: LogicalProject(info.city=[ITEM(GEOIP('my-datasource':VARCHAR, $0), 'city')]) CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]]) physical: | - CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), info.city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"info.city":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQErXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJJVEVNIiwKICAgICJraW5kIjogIklURU0iLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiR0VPSVAiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ1ZHQiOiAiRVhQUl9JUCIsCiAgICAgICAgICAgICJ0eXBlIjogIk9USEVSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJNQVAiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJrZXkiOiB7CiAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgfSwKICAgICAgICAidmFsdWUiOiB7CiAgICAgICAgICAidHlwZSI6ICJBTlkiLAogICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICAgICAic2NhbGUiOiAtMjE0NzQ4MzY0OAogICAgICAgIH0KICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IDQKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[2,0,2],"DIGESTS":["my-datasource","host","city"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), info.city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"info.city":{"terms":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXQErXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJJVEVNIiwKICAgICJraW5kIjogIklURU0iLAogICAgInN5bnRheCI6ICJTUEVDSUFMIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiR0VPSVAiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImR5bmFtaWNQYXJhbSI6IDAsCiAgICAgICAgICAidHlwZSI6IHsKICAgICAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJkeW5hbWljUGFyYW0iOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ1ZHQiOiAiRVhQUl9JUCIsCiAgICAgICAgICAgICJ0eXBlIjogIk9USEVSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJNQVAiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJrZXkiOiB7CiAgICAgICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgICAgfSwKICAgICAgICAidmFsdWUiOiB7CiAgICAgICAgICAidHlwZSI6ICJBTlkiLAogICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICAgICAic2NhbGUiOiAtMjE0NzQ4MzY0OAogICAgICAgIH0KICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAiZHluYW1pY1BhcmFtIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IDQKICAgICAgfQogICAgfQogIF0KfQ==\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0,"SOURCES":[2,0,2],"DIGESTS":["my-datasource","host","city"]}},"missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=10000, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_append_command.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_append_command.json deleted file mode 100644 index 5665e159573..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_append_command.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalUnion(all=[true])\n LogicalProject(cnt=[$1], gender=[$0])\n LogicalAggregate(group=[{0}], cnt=[COUNT($1)])\n LogicalProject(gender=[$4], balance=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(cnt=[$0], gender=[null:VARCHAR])\n LogicalAggregate(group=[{}], cnt=[COUNT()])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableUnion(all=[true])\n EnumerableCalc(expr#0..1=[{inputs}], cnt=[$t1], gender=[$t0])\n EnumerableAggregate(group=[{4}], cnt=[COUNT($7)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[null:VARCHAR], proj#0..1=[{exprs}])\n EnumerableAggregate(group=[{}], cnt=[COUNT()])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_append_command.yaml b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_append_command.yaml new file mode 100644 index 00000000000..eb4b6f09d58 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_append_command.yaml @@ -0,0 +1,20 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalUnion(all=[true]) + LogicalProject(cnt=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], cnt=[COUNT($1)]) + LogicalProject(gender=[$4], balance=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + LogicalProject(cnt=[$0], gender=[null:VARCHAR]) + LogicalAggregate(group=[{}], cnt=[COUNT()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableUnion(all=[true]) + EnumerableCalc(expr#0..1=[{inputs}], cnt=[$t1], gender=[$t0]) + EnumerableAggregate(group=[{4}], cnt=[COUNT($7)]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) + EnumerableCalc(expr#0=[{inputs}], expr#1=[null:VARCHAR], proj#0..1=[{exprs}]) + EnumerableAggregate(group=[{}], cnt=[COUNT()]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp.yaml index e57913c2f62..c2b045b4600 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp.yaml @@ -11,6 +11,5 @@ root: ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ :[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\",\"missing\":\"_first\"\ - }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + }}]}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp_can_match_shortcut.yaml index b0a8deed278..0e6f5549a56 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp_can_match_shortcut.yaml @@ -15,6 +15,6 @@ root: :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ - asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + asc\",\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp_no_can_match_shortcut.yaml index b0a8deed278..0e6f5549a56 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp_no_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_timestamp_no_can_match_shortcut.yaml @@ -15,6 +15,6 @@ root: :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ - asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + asc\",\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_with_after_timestamp.yaml index e57913c2f62..c2b045b4600 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_with_after_timestamp.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/asc_sort_with_after_timestamp.yaml @@ -11,6 +11,5 @@ root: ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ :[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\",\"missing\":\"_first\"\ - }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + }}]}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_high.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_high.yaml index ffe939e5a52..ec60aa78ed7 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_high.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_high.yaml @@ -7,6 +7,6 @@ root: description: request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(`agent.name`)\":{\"cardinality\"\ - :{\"field\":\"agent.name\"}}}}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + :{\"field\":\"agent.name\"}}}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_high_2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_high_2.yaml index 0c147949642..996fc82b1b7 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_high_2.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_high_2.yaml @@ -7,6 +7,6 @@ root: description: request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(`event.id`)\":{\"cardinality\"\ - :{\"field\":\"event.id\"}}}}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + :{\"field\":\"event.id\"}}}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_low.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_low.yaml index f064201008e..90117e88947 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_low.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/cardinality_agg_low.yaml @@ -7,6 +7,6 @@ root: description: request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"from\":0,\"\ size\":0,\"timeout\":\"1m\",\"aggregations\":{\"dc(`cloud.region`)\":{\"\ - cardinality\":{\"field\":\"cloud.region\"}}}}, needClean=true, searchDone=false,\ - \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + cardinality\":{\"field\":\"cloud.region\"}}}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_date_histogram_daily.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_date_histogram_daily.yaml index 073078ddf0d..c0f27278561 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_date_histogram_daily.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_date_histogram_daily.yaml @@ -20,6 +20,6 @@ root: sources\":[{\"span(`@timestamp`,1d)\":{\"date_histogram\":{\"field\"\ :\"@timestamp\",\"missing_bucket\":false,\"order\":\"asc\",\"fixed_interval\"\ :\"1d\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\"\ - :\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + :\"_index\"}}}}}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms.yaml index 6a0e0c660da..a032981995e 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms.yaml @@ -21,7 +21,6 @@ root: ,\"missing_bucket\":true,\"missing_order\":\"last\",\"order\":\"desc\"\ }}},{\"cloud.region\":{\"terms\":{\"field\":\"cloud.region\",\"missing_bucket\"\ :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ - :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms_keyword.yaml index 5c77f33d0cd..43017baa00f 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms_keyword.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/composite_terms_keyword.yaml @@ -23,7 +23,6 @@ root: :true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"aws.cloudwatch.log_stream\"\ :{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\",\"missing_bucket\"\ :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ - :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/date_histogram_hourly_agg.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/date_histogram_hourly_agg.yaml index 72549142297..f294eb043f3 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/date_histogram_hourly_agg.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/date_histogram_hourly_agg.yaml @@ -10,6 +10,6 @@ root: composite\":{\"size\":1000,\"sources\":[{\"span(`@timestamp`,1h)\":{\"date_histogram\"\ :{\"field\":\"@timestamp\",\"missing_bucket\":false,\"order\":\"asc\",\"\ fixed_interval\":\"1h\"}}}]},\"aggregations\":{\"count()\":{\"value_count\"\ - :{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + :{\"field\":\"_index\"}}}}}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/date_histogram_minute_agg.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/date_histogram_minute_agg.yaml index be30d2a0801..32e0cdae956 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/date_histogram_minute_agg.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/date_histogram_minute_agg.yaml @@ -14,6 +14,6 @@ root: :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(`@timestamp`,1m)\"\ :{\"date_histogram\":{\"field\":\"@timestamp\",\"missing_bucket\":false,\"\ order\":\"asc\",\"fixed_interval\":\"1m\"}}}]},\"aggregations\":{\"count()\"\ - :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ - \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + :{\"value_count\":{\"field\":\"_index\"}}}}}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/default.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/default.yaml index 23ca821adf6..28d71ecf09e 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/default.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/default.yaml @@ -10,6 +10,5 @@ root: size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ - :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + :[]}}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp.yaml index ed13e6905cb..16f1fcfa33a 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp.yaml @@ -11,6 +11,5 @@ root: ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ :[]},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"missing\":\"_last\"\ - }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + }}]}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp_can_match_shortcut.yaml index e2fc446cdd7..8f8424df6b7 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp_can_match_shortcut.yaml @@ -15,6 +15,6 @@ root: :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ - desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + desc\",\"missing\":\"_last\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp_no_can_match_shortcut.yaml index e2fc446cdd7..8f8424df6b7 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp_no_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_timestamp_no_can_match_shortcut.yaml @@ -15,6 +15,6 @@ root: :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"\ - desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + desc\",\"missing\":\"_last\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_with_after_timestamp.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_with_after_timestamp.yaml index ed13e6905cb..16f1fcfa33a 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_with_after_timestamp.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/desc_sort_with_after_timestamp.yaml @@ -11,6 +11,5 @@ root: ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ :[]},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"missing\":\"_last\"\ - }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + }}]}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_in_range.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_in_range.yaml index 2a85a0971ce..1ec8b25e62b 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_in_range.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_in_range.yaml @@ -20,6 +20,5 @@ root: :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"agent\",\"process\",\"\ log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ - :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + :[]}}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_terms.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_terms.yaml index 3031899608b..1fd8d271516 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_terms.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_terms.yaml @@ -19,7 +19,6 @@ root: :{\"composite\":{\"size\":1000,\"sources\":[{\"aws.cloudwatch.log_stream\"\ :{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\",\"missing_bucket\"\ :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ - :{\"station\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + :{\"station\":{\"value_count\":{\"field\":\"_index\"}}}}}}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_terms_low_cardinality.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_terms_low_cardinality.yaml index 2a05fec4f3e..4fd71c39b96 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_terms_low_cardinality.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/keyword_terms_low_cardinality.yaml @@ -19,7 +19,6 @@ root: :{\"composite\":{\"size\":1000,\"sources\":[{\"aws.cloudwatch.log_stream\"\ :{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\",\"missing_bucket\"\ :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ - :{\"country\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + :{\"country\":{\"value_count\":{\"field\":\"_index\"}}}}}}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/multi_terms_keyword.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/multi_terms_keyword.yaml index 80709a3e0f5..2954c1a0fa7 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/multi_terms_keyword.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/multi_terms_keyword.yaml @@ -25,7 +25,6 @@ root: ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"\ }}},{\"cloud.region\":{\"terms\":{\"field\":\"cloud.region\",\"missing_bucket\"\ :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ - :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message.yaml index 20f024800e5..aa99c0c8636 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message.yaml @@ -15,6 +15,6 @@ root: :true,\"fuzzy_transpositions\":true,\"boost\":1.0}},\"_source\":{\"includes\"\ :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ - aws\",\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + aws\",\"event\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message_filtered.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message_filtered.yaml index fdd6d08721b..069db10a79d 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message_filtered.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message_filtered.yaml @@ -21,6 +21,5 @@ root: :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"agent\",\"process\",\"\ log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ - :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + :[]}}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message_filtered_sorted_num.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message_filtered_sorted_num.yaml index 4cc79a8db95..e3ba107ee04 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message_filtered_sorted_num.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/query_string_on_message_filtered_sorted_num.yaml @@ -22,6 +22,5 @@ root: log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ :[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\",\"missing\":\"_first\"\ - }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + }}]}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range.yaml index f9d406b2906..4406c961892 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range.yaml @@ -14,6 +14,6 @@ root: boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\"\ :{\"includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ - ,\"aws\",\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ - \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + ,\"aws\",\"event\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_agg_1.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_agg_1.yaml index 3f99d51799a..40df7fa28c9 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_agg_1.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_agg_1.yaml @@ -22,7 +22,6 @@ root: - name: OpenSearchIndexScan description: request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ - from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false,\ - \ pitId=*,\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_agg_2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_agg_2.yaml index 400198d6635..fba6186ffd3 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_agg_2.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_agg_2.yaml @@ -20,7 +20,6 @@ root: - name: OpenSearchIndexScan description: request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ - from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true, searchDone=false,\ - \ pitId=*,\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_auto_date_histo.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_auto_date_histo.yaml index 159c9be49a1..b1717e04211 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_auto_date_histo.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_auto_date_histo.yaml @@ -32,7 +32,6 @@ root: - name: OpenSearchIndexScan description: request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ - from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ - \ searchDone=false, pitId=*,\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_auto_date_histo_with_metrics.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_auto_date_histo_with_metrics.yaml index 4f3004af106..b146d7e284c 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_auto_date_histo_with_metrics.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_auto_date_histo_with_metrics.yaml @@ -30,7 +30,6 @@ root: - name: OpenSearchIndexScan description: request: "OpenSearchQueryRequest(indexName=big5, sourceBuilder={\"\ - from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ - \ searchDone=false, pitId=*,\ + from\":0,\"size\":10000,\"timeout\":\"1m\"}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_big_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_big_range_big_term_query.yaml index 2663492036c..44ffe43c5c7 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_big_range_big_term_query.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_big_range_big_term_query.yaml @@ -16,6 +16,6 @@ root: adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ - ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + ,\"event\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_small_range_big_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_small_range_big_term_query.yaml index 3365a5b0813..c17a49a4c2e 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_small_range_big_term_query.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_small_range_big_term_query.yaml @@ -14,6 +14,5 @@ root: :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"agent\",\"process\",\"\ log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\",\"\ data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ - :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + :[]}}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_small_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_small_range_small_term_query.yaml index 664d6428c7b..d04271ee003 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_small_range_small_term_query.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_conjunction_small_range_small_term_query.yaml @@ -16,6 +16,6 @@ root: adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ - ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + ,\"event\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_disjunction_big_range_small_term_query.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_disjunction_big_range_small_term_query.yaml index 641befc2867..c0d63da12bf 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_disjunction_big_range_small_term_query.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_field_disjunction_big_range_small_term_query.yaml @@ -16,6 +16,6 @@ root: adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ - ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + ,\"event\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_numeric.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_numeric.yaml index 156f9ced9fe..5d49015ec62 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_numeric.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_numeric.yaml @@ -14,6 +14,6 @@ root: adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"\ agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"\ @timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\"\ - ,\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + ,\"event\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_with_asc_sort.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_with_asc_sort.yaml index 05a16cc76cf..0681c881ce9 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_with_asc_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_with_asc_sort.yaml @@ -15,6 +15,6 @@ root: :{\"includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ ,\"aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\"\ - :\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + :\"asc\",\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_with_desc_sort.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_with_desc_sort.yaml index e7322cb282e..1b9d3f2c246 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/range_with_desc_sort.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/range_with_desc_sort.yaml @@ -15,6 +15,6 @@ root: :{\"includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ ,\"aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\"\ - :\"desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + :\"desc\",\"missing\":\"_last\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/scroll.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/scroll.yaml index 23ca821adf6..28d71ecf09e 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/scroll.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/scroll.yaml @@ -10,6 +10,5 @@ root: size\":10,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"agent\",\"process\"\ ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ - :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + :[]}}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_keyword_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_keyword_can_match_shortcut.yaml index 20b18df0256..926319755f8 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_keyword_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_keyword_can_match_shortcut.yaml @@ -15,6 +15,6 @@ root: :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"meta.file\":{\"order\":\"\ - asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + asc\",\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_keyword_no_can_match_shortcut.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_keyword_no_can_match_shortcut.yaml index 20b18df0256..926319755f8 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_keyword_no_can_match_shortcut.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_keyword_no_can_match_shortcut.yaml @@ -15,6 +15,6 @@ root: :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"meta.file\":{\"order\":\"\ - asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + asc\",\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_asc.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_asc.yaml index 9d0f5c0ab0c..10228036927 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_asc.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_asc.yaml @@ -11,6 +11,5 @@ root: ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ :[]},\"sort\":[{\"metrics.size\":{\"order\":\"asc\",\"missing\":\"_first\"\ - }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + }}]}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_asc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_asc_with_match.yaml index 3718496568c..1925583a9e0 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_asc_with_match.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_asc_with_match.yaml @@ -16,6 +16,6 @@ root: :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"metrics.size\":{\"order\"\ - :\"asc\",\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + :\"asc\",\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_desc.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_desc.yaml index 27126b931f0..acf47c582d9 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_desc.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_desc.yaml @@ -11,6 +11,5 @@ root: ,\"log\",\"message\",\"tags\",\"cloud\",\"input\",\"@timestamp\",\"ecs\"\ ,\"data_stream\",\"meta\",\"host\",\"metrics\",\"aws\",\"event\"],\"excludes\"\ :[]},\"sort\":[{\"metrics.size\":{\"order\":\"desc\",\"missing\":\"_last\"\ - }}]}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + }}]}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_desc_with_match.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_desc_with_match.yaml index a146d0531d5..00b13e06628 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_desc_with_match.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/sort_numeric_desc_with_match.yaml @@ -16,6 +16,6 @@ root: :[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\",\"input\"\ ,\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\",\"\ aws\",\"event\"],\"excludes\":[]},\"sort\":[{\"metrics.size\":{\"order\"\ - :\"desc\",\"missing\":\"_last\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + :\"desc\",\"missing\":\"_last\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/term.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/term.yaml index ea9331ffa08..56f97a7481e 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/term.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/term.yaml @@ -11,6 +11,6 @@ root: value\":\"/var/log/messages/birdknight\",\"boost\":1.0}}},\"_source\":{\"\ includes\":[\"agent\",\"process\",\"log\",\"message\",\"tags\",\"cloud\"\ ,\"input\",\"@timestamp\",\"ecs\",\"data_stream\",\"meta\",\"host\",\"metrics\"\ - ,\"aws\",\"event\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ - \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + ,\"aws\",\"event\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/terms_significant_1.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/terms_significant_1.yaml index 09bd047cfb9..a3da18d850c 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/terms_significant_1.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/terms_significant_1.yaml @@ -21,7 +21,6 @@ root: ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"\ }}},{\"process.name\":{\"terms\":{\"field\":\"process.name\",\"missing_bucket\"\ :true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ - :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + :{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/big5/terms_significant_2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/big5/terms_significant_2.yaml index 908ab96f04b..c15fd3db7b0 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/big5/terms_significant_2.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/big5/terms_significant_2.yaml @@ -22,6 +22,5 @@ root: }}},{\"aws.cloudwatch.log_stream\":{\"terms\":{\"field\":\"aws.cloudwatch.log_stream\"\ ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"\ }}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"\ - }}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + }}}}}}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.yaml index 454ce35f3a8..73249d4d9ff 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_agg_push.yaml @@ -13,6 +13,6 @@ root: ,\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"\ city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"\ missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\"\ - :{\"avg\":{\"field\":\"age\"}}}}}}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + :{\"avg\":{\"field\":\"age\"}}}}}}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_date_string.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_date_string.yaml index 289c6297e6f..5bf06c10265 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_date_string.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_date_string.yaml @@ -16,7 +16,6 @@ root: description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_date_formats,\ \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"\ - _source\":{\"includes\":[\"yyyy-MM-dd\"],\"excludes\":[]}}, needClean=true,\ - \ searchDone=false, pitId=*,\ + _source\":{\"includes\":[\"yyyy-MM-dd\"],\"excludes\":[]}}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_time_string.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_time_string.yaml index c4b8521567a..0a06e142431 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_time_string.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_time_string.yaml @@ -16,7 +16,6 @@ root: description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_date_formats,\ \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"\ - _source\":{\"includes\":[\"custom_time\"],\"excludes\":[]}}, needClean=true,\ - \ searchDone=false, pitId=*,\ + _source\":{\"includes\":[\"custom_time\"],\"excludes\":[]}}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_timestamp_string.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_timestamp_string.yaml index 5eb98d3ca33..778fdeaa1c7 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_timestamp_string.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_push_compare_timestamp_string.yaml @@ -14,7 +14,6 @@ root: include_lower\":false,\"include_upper\":true,\"boost\":1.0}}}],\"adjust_pure_negative\"\ :true,\"boost\":1.0}},\"_source\":{\"includes\":[\"account_number\",\"firstname\"\ ,\"address\",\"birthdate\",\"gender\",\"city\",\"lastname\",\"balance\"\ - ,\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]}}, needClean=true,\ - \ searchDone=false, pitId=*,\ + ,\"employer\",\"state\",\"age\",\"email\",\"male\"],\"excludes\":[]}}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_then_limit_push.yaml index f0f18391b93..66c7729f993 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_then_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_filter_then_limit_push.yaml @@ -9,6 +9,5 @@ root: \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"query\":{\"\ range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\"\ :true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\"\ - :[]}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null,\ - \ searchAfter=null, searchResponse=null)" + :[]}}, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function_case_insensitive.yaml index 5b950289224..ff157e036d0 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_keyword_like_function_case_insensitive.yaml @@ -11,7 +11,6 @@ root: :{\"wildcard\":{\"firstname.keyword\":{\"wildcard\":\"*mbe*\",\"case_insensitive\"\ :true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"account_number\",\"\ firstname\",\"address\",\"balance\",\"gender\",\"city\",\"employer\",\"\ - state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, needClean=true,\ - \ searchDone=false, pitId=*,\ + state\",\"age\",\"email\",\"lastname\"],\"excludes\":[]}}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_5_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_5_push.yaml index 604ec122fdf..7ce50a2b37c 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_5_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_5_push.yaml @@ -7,6 +7,6 @@ root: description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ - includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ - \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + includes\":[\"age\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_filter_5_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_filter_5_push.yaml index d1ba3213130..28b63f58511 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_filter_5_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10_filter_5_push.yaml @@ -15,7 +15,6 @@ root: - name: OpenSearchIndexScan description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ - \ sourceBuilder={\"from\":0,\"size\":10,\"timeout\":\"1m\"}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + \ sourceBuilder={\"from\":0,\"size\":10,\"timeout\":\"1m\"}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10from1_10from2_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10from1_10from2_push.yaml index c6fbc4fd672..53ed94585f0 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10from1_10from2_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_10from1_10from2_push.yaml @@ -7,6 +7,6 @@ root: description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ \ sourceBuilder={\"from\":3,\"size\":8,\"timeout\":\"1m\",\"_source\":{\"\ - includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ - \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + includes\":[\"age\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_5_10_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_5_10_push.yaml index 604ec122fdf..7ce50a2b37c 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_5_10_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_5_10_push.yaml @@ -7,6 +7,6 @@ root: description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ - includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ - \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + includes\":[\"age\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_offsets_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_offsets_push.yaml index 17960c2d39b..1d4cd474476 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_offsets_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_offsets_push.yaml @@ -7,6 +7,6 @@ root: description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ \ sourceBuilder={\"from\":3,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ - includes\":[\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ - \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + includes\":[\"age\"],\"excludes\":[]}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.yaml index 2d14d36d922..d2be3d3d5d2 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_push.yaml @@ -11,7 +11,6 @@ root: - name: OpenSearchIndexScan description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ - \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_filter_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_filter_push.yaml index f0f312cc4cd..221062310b3 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_filter_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_filter_push.yaml @@ -10,7 +10,6 @@ root: - name: OpenSearchIndexScan description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ - \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_sort_push.yaml index a49d48e5aa4..f2daa15d506 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_limit_then_sort_push.yaml @@ -8,6 +8,6 @@ root: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\"\ - ,\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + ,\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_output.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_output.yaml index 51065230f90..db9b43af5bf 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_output.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_output.yaml @@ -30,7 +30,6 @@ root: :true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\"\ :{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"\ missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\"\ - :{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + :{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern.yaml index ec791b34e14..517435c9a92 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern.yaml @@ -10,7 +10,6 @@ root: \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\"\ :{\"includes\":[\"account_number\",\"firstname\",\"address\",\"balance\"\ ,\"gender\",\"city\",\"employer\",\"state\",\"age\",\"email\",\"lastname\"\ - ,\"patterns_field\"],\"excludes\":[]}}, needClean=true, searchDone=false,\ - \ pitId=*,\ + ,\"patterns_field\"],\"excludes\":[]}}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml index 15f29f544ae..ea56b054bb2 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_patterns_simple_pattern_agg_push.yaml @@ -14,7 +14,6 @@ root: missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\"\ :{\"value_count\":{\"field\":\"_index\"}},\"sample_logs\":{\"top_hits\"\ :{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"\ - explain\":false,\"fields\":[{\"field\":\"email\"}]}}}}}}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + explain\":false,\"fields\":[{\"field\":\"email\"}]}}}}}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_count_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_count_push.yaml index a49d48e5aa4..f2daa15d506 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_count_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_count_push.yaml @@ -8,6 +8,6 @@ root: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\"\ - ,\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + ,\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_limit_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_limit_push.yaml index a49d48e5aa4..f2daa15d506 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_limit_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_sort_then_limit_push.yaml @@ -8,6 +8,6 @@ root: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"_source\":{\"\ includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"asc\"\ - ,\"missing\":\"_first\"}}]}, needClean=true, searchDone=false, pitId=*,\ - \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + ,\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=null, searchAfter=null,\ + \ searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.yaml index 3fd26cb19a6..1277837696d 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.yaml @@ -10,6 +10,6 @@ root: :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1m)\"\ :{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"\ order\":\"asc\",\"fixed_interval\":\"1m\"}}}]},\"aggregations\":{\"count()\"\ - :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ - \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + :{\"value_count\":{\"field\":\"_index\"}}}}}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.yaml index 167328625f1..017c998a042 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.yaml @@ -10,6 +10,6 @@ root: :{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1M)\"\ :{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"\ order\":\"asc\",\"calendar_interval\":\"1M\"}}}]},\"aggregations\":{\"count()\"\ - :{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false,\ - \ pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + :{\"value_count\":{\"field\":\"_index\"}}}}}}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_take.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_take.yaml index 7e4de15c4d4..4497e8e6d63 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_take.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_take.yaml @@ -8,7 +8,6 @@ root: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ \ sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\"\ :{\"take\":{\"top_hits\":{\"from\":0,\"size\":2,\"version\":false,\"seq_no_primary_term\"\ - :false,\"explain\":false,\"fields\":[{\"field\":\"firstname\"}]}}}}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + :false,\"explain\":false,\"fields\":[{\"field\":\"firstname\"}]}}}}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function.yaml index 2433ee66003..aff2f01857c 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function.yaml @@ -11,7 +11,6 @@ root: - name: OpenSearchIndexScan description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ - \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ - \ searchDone=false, pitId=*,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function_case_insensitive.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function_case_insensitive.yaml index 3187fd69900..2130b25c0e5 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function_case_insensitive.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_text_like_function_case_insensitive.yaml @@ -11,7 +11,6 @@ root: - name: OpenSearchIndexScan description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ - \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, needClean=true,\ - \ searchDone=false, pitId=*,\ + \ sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\"}, pitId=*,\ \ cursorKeepAlive=1m, searchAfter=null, searchResponse=null)" children: [] \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.yaml index 6a4ccc5b633..e6157d8ed2a 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_push.yaml @@ -14,7 +14,6 @@ root: - name: OpenSearchIndexScan description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ - \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\"}, pitId=*,\ + \ cursorKeepAlive=null, searchAfter=null, searchResponse=null)" children: [] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.yaml b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.yaml index 47366bfd683..e246a328965 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.yaml +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_trendline_sort_push.yaml @@ -15,7 +15,6 @@ root: description: request: "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account,\ \ sourceBuilder={\"from\":0,\"size\":5,\"timeout\":\"1m\",\"sort\":[{\"\ - age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, needClean=true,\ - \ searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null,\ - \ searchResponse=null)" + age\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, pitId=*, cursorKeepAlive=null,\ + \ searchAfter=null, searchResponse=null)" children: [] diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index b52d4a06820..620c9b97cf7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -136,7 +136,7 @@ public static class AggregateBuilderHelper { final Map fieldTypes; final RelOptCluster cluster; final boolean bucketNullable; - final int bucketSize; + final int queryBucketSize; > T build(RexNode node, T aggBuilder) { return build(node, aggBuilder::field, aggBuilder::script); @@ -264,7 +264,8 @@ public static Pair, OpenSearchAggregationResponseParser + " aggregation"); } AggregationBuilder compositeBuilder = - AggregationBuilders.composite("composite_buckets", buckets).size(helper.bucketSize); + AggregationBuilders.composite("composite_buckets", buckets) + .size(helper.queryBucketSize); if (subBuilder != null) { compositeBuilder.subAggregations(subBuilder); } @@ -763,7 +764,7 @@ private static ValuesSourceAggregationBuilder createTermsAggregationBuilder( helper.build( group, new TermsAggregationBuilder(bucketName) - .size(helper.bucketSize) + .size(helper.queryBucketSize) .order(BucketOrder.key(true))); return withValueTypeHint( sourceBuilder, diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index 7a5bc830d05..04af888cfdd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -12,11 +12,15 @@ import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.TestOnly; import org.opensearch.action.search.*; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -28,6 +32,8 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.SearchModule; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.InternalComposite; import org.opensearch.search.builder.PointInTimeBuilder; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.sort.FieldSortBuilder; @@ -48,6 +54,7 @@ @Getter @ToString public class OpenSearchQueryRequest implements OpenSearchRequest { + private static final Logger LOG = LogManager.getLogger(); /** {@link OpenSearchRequest.IndexName}. */ private final IndexName indexName; @@ -62,51 +69,63 @@ public class OpenSearchQueryRequest implements OpenSearchRequest { /** List of includes expected in the response. */ @EqualsAndHashCode.Exclude @ToString.Exclude private final List includes; - @EqualsAndHashCode.Exclude private boolean needClean = true; + @EqualsAndHashCode.Exclude @ToString.Exclude private boolean needClean = true; /** Indicate the search already done. */ - private boolean searchDone = false; + @EqualsAndHashCode.Exclude @ToString.Exclude private boolean searchDone = false; private String pitId; - private TimeValue cursorKeepAlive; + private final TimeValue cursorKeepAlive; private Object[] searchAfter; private SearchResponse searchResponse = null; - /** Constructor of OpenSearchQueryRequest. */ - public OpenSearchQueryRequest( - String indexName, int size, OpenSearchExprValueFactory factory, List includes) { - this(new IndexName(indexName), size, factory, includes); - } + @ToString.Exclude private Map afterKey; - /** Constructor of OpenSearchQueryRequest. */ - public OpenSearchQueryRequest( - IndexName indexName, int size, OpenSearchExprValueFactory factory, List includes) { - this.indexName = indexName; - this.sourceBuilder = new SearchSourceBuilder(); + @TestOnly + static OpenSearchQueryRequest of( + String indexName, int size, OpenSearchExprValueFactory factory, List includes) { + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); sourceBuilder.from(0); sourceBuilder.size(size); sourceBuilder.timeout(DEFAULT_QUERY_TIMEOUT); - this.exprValueFactory = factory; - this.includes = includes; + return of(indexName, sourceBuilder, factory, includes); } - /** Constructor of OpenSearchQueryRequest. */ - public OpenSearchQueryRequest( + @TestOnly + static OpenSearchQueryRequest of( + String indexName, + SearchSourceBuilder sourceBuilder, + OpenSearchExprValueFactory factory, + List includes) { + return OpenSearchQueryRequest.of(new IndexName(indexName), sourceBuilder, factory, includes); + } + + /** Build an OpenSearchQueryRequest without PIT support. */ + public static OpenSearchQueryRequest of( IndexName indexName, SearchSourceBuilder sourceBuilder, OpenSearchExprValueFactory factory, List includes) { - this.indexName = indexName; - this.sourceBuilder = sourceBuilder; - this.exprValueFactory = factory; - this.includes = includes; + return new OpenSearchQueryRequest(indexName, sourceBuilder, factory, includes, null, null); } - /** Constructor of OpenSearchQueryRequest with PIT support. */ - public OpenSearchQueryRequest( + /** Build an OpenSearchQueryRequest with PIT support. */ + public static OpenSearchQueryRequest pitOf( + IndexName indexName, + SearchSourceBuilder sourceBuilder, + OpenSearchExprValueFactory factory, + List includes, + TimeValue cursorKeepAlive, + String pitId) { + return new OpenSearchQueryRequest( + indexName, sourceBuilder, factory, includes, cursorKeepAlive, pitId); + } + + /** Do not new it directly, use of() and pitOf() instead. */ + OpenSearchQueryRequest( IndexName indexName, SearchSourceBuilder sourceBuilder, OpenSearchExprValueFactory factory, @@ -171,27 +190,56 @@ public OpenSearchResponse search( Function searchAction, Function scrollAction) { if (this.pitId == null) { - // When SearchRequest doesn't contain PitId, fetch single page request - if (searchDone) { - return new OpenSearchResponse( - SearchHits.empty(), exprValueFactory, includes, isCountAggRequest()); - } else { - // get the value before set searchDone = true - boolean isCountAggRequest = isCountAggRequest(); - searchDone = true; - return new OpenSearchResponse( - searchAction.apply( - new SearchRequest().indices(indexName.getIndexNames()).source(sourceBuilder)), - exprValueFactory, - includes, - isCountAggRequest); - } + return search(searchAction); } else { // Search with PIT instead of scroll API return searchWithPIT(searchAction); } } + private OpenSearchResponse search(Function searchAction) { + OpenSearchResponse openSearchResponse; + if (searchDone) { + openSearchResponse = + new OpenSearchResponse( + SearchHits.empty(), exprValueFactory, includes, isCountAggRequest()); + } else { + // Set afterKey to request, null for first round (afterKey is null in the beginning). + if (this.sourceBuilder.aggregations() != null) { + this.sourceBuilder.aggregations().getAggregatorFactories().stream() + .filter(a -> a instanceof CompositeAggregationBuilder) + .forEach(c -> ((CompositeAggregationBuilder) c).aggregateAfter(afterKey)); + if (LOG.isDebugEnabled()) { + LOG.debug(sourceBuilder); + } + } + + SearchRequest searchRequest = + new SearchRequest().indices(indexName.getIndexNames()).source(this.sourceBuilder); + this.searchResponse = searchAction.apply(searchRequest); + + openSearchResponse = + new OpenSearchResponse( + this.searchResponse, exprValueFactory, includes, isCountAggRequest()); + + // Update afterKey from response + if (openSearchResponse.isAggregationResponse()) { + openSearchResponse.getAggregations().asList().stream() + .filter(a -> a instanceof InternalComposite) + .forEach(c -> afterKey = ((InternalComposite) c).afterKey()); + } + if (afterKey != null) { + // For composite aggregation, searchDone is determined by response result. + searchDone = openSearchResponse.isEmpty(); + } else { + // Direct set searchDone to true for non-composite aggregation. + searchDone = true; + } + needClean = searchDone; + } + return openSearchResponse; + } + public OpenSearchResponse searchWithPIT(Function searchAction) { OpenSearchResponse openSearchResponse; if (searchDone) { @@ -237,6 +285,9 @@ public OpenSearchResponse searchWithPIT(Function if (searchHits != null && searchHits.length > 0) { searchAfter = searchHits[searchHits.length - 1].getSortValues(); this.sourceBuilder.searchAfter(searchAfter); + if (LOG.isDebugEnabled()) { + LOG.debug(sourceBuilder); + } } } return openSearchResponse; @@ -252,6 +303,8 @@ public void clean(Consumer cleanAction) { } } finally { this.pitId = null; + this.searchAfter = null; + this.afterKey = null; } } @@ -264,6 +317,8 @@ public void forceClean(Consumer cleanAction) { } } finally { this.pitId = null; + this.searchAfter = null; + this.afterKey = null; } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java index 5c3ecd11e49..435cef22ef4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java @@ -24,6 +24,7 @@ import lombok.ToString; import org.apache.commons.lang3.tuple.Pair; import org.apache.lucene.search.join.ScoreMode; +import org.jetbrains.annotations.TestOnly; import org.opensearch.action.search.CreatePitRequest; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.query.BoolQueryBuilder; @@ -71,9 +72,7 @@ public class OpenSearchRequestBuilder { private int startFrom = 0; - @ToString.Exclude private final Settings settings; - - @ToString.Exclude private boolean topHitsAgg = false; + @EqualsAndHashCode.Exclude @ToString.Exclude private final Settings settings; public static class PushDownUnSupportedException extends RuntimeException { public PushDownUnSupportedException(String message) { @@ -98,7 +97,10 @@ public OpenSearchRequestBuilder( * Build DSL request. * * @return query request with PIT or scroll request + * @deprecated for testing only now. */ + @TestOnly + @Deprecated public OpenSearchRequest build( OpenSearchRequest.IndexName indexName, TimeValue cursorKeepAlive, OpenSearchClient client) { return build(indexName, cursorKeepAlive, client, false); @@ -114,7 +116,7 @@ public OpenSearchRequest build( * 2. If mapping is empty. It means no data in the index. PIT search relies on `_id` fields to do sort, thus it will fail if using PIT search in this case. */ if (sourceBuilder.size() == 0 || isMappingEmpty) { - return new OpenSearchQueryRequest(indexName, sourceBuilder, exprValueFactory, List.of()); + return OpenSearchQueryRequest.of(indexName, sourceBuilder, exprValueFactory, List.of()); } return buildRequestWithPit(indexName, cursorKeepAlive, client); } @@ -130,13 +132,13 @@ private OpenSearchRequest buildRequestWithPit( sourceBuilder.size(maxResultWindow - startFrom); // Search with PIT request String pitId = createPit(indexName, cursorKeepAlive, client); - return new OpenSearchQueryRequest( + return OpenSearchQueryRequest.pitOf( indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId); } else { sourceBuilder.from(startFrom); sourceBuilder.size(size); // Search with non-Pit request - return new OpenSearchQueryRequest(indexName, sourceBuilder, exprValueFactory, includes); + return OpenSearchQueryRequest.of(indexName, sourceBuilder, exprValueFactory, includes); } } else { if (startFrom != 0) { @@ -145,7 +147,7 @@ private OpenSearchRequest buildRequestWithPit( sourceBuilder.size(pageSize); // Search with PIT request String pitId = createPit(indexName, cursorKeepAlive, client); - return new OpenSearchQueryRequest( + return OpenSearchQueryRequest.pitOf( indexName, sourceBuilder, exprValueFactory, includes, cursorKeepAlive, pitId); } } @@ -239,6 +241,16 @@ public void pushDownSortSuppliers(List>> sortBuilderSupp } } + /** Push down the limit to requestedTotalSize for paginating aggregation. */ + public void pushDownLimitToRequestTotal(Integer limit, Integer offset) { + requestedTotalSize = Math.min(requestedTotalSize, limit + offset); + } + + /** Reset the requestedTotalSize since we convert composite aggregation to others. */ + public void resetRequestTotal() { + requestedTotalSize = Integer.MAX_VALUE; + } + public void pushDownLimit(Integer limit, Integer offset) { // If there are multiple limit, we take the minimum among them // E.g. for `source=t | head 10 | head 5`, we take 5 diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java index be5c1f0baf9..4cf9a868116 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java @@ -60,7 +60,7 @@ public class OpenSearchScrollRequest implements OpenSearchRequest { public static final String NO_SCROLL_ID = ""; - @EqualsAndHashCode.Exclude private boolean needClean = true; + @EqualsAndHashCode.Exclude @ToString.Exclude private boolean needClean = true; @Getter @EqualsAndHashCode.Exclude @ToString.Exclude private final List includes; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java index e35c7efcf7b..0a47dc64a5e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java @@ -20,12 +20,15 @@ import java.util.Map; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; +import lombok.Getter; import lombok.ToString; +import org.jetbrains.annotations.TestOnly; import org.opensearch.action.search.SearchResponse; import org.opensearch.core.common.text.Text; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregations; +import org.opensearch.search.aggregations.bucket.composite.InternalComposite; import org.opensearch.sql.data.model.ExprFloatValue; import org.opensearch.sql.data.model.ExprLongValue; import org.opensearch.sql.data.model.ExprStringValue; @@ -39,11 +42,13 @@ @ToString public class OpenSearchResponse implements Iterable { + public static final OpenSearchResponse EMPTY = empty(); + /** Search query result (non-aggregation). */ private final SearchHits hits; /** Search aggregation result. */ - private final Aggregations aggregations; + @Getter private final Aggregations aggregations; /** List of requested include fields. */ private final List includes; @@ -53,12 +58,17 @@ public class OpenSearchResponse implements Iterable { /** OpenSearchExprValueFactory used to build ExprValue from search result. */ @EqualsAndHashCode.Exclude private final OpenSearchExprValueFactory exprValueFactory; - /** Constructor of OpenSearchResponse. */ - public OpenSearchResponse( + /** The empty OpenSearchResponse which is used for invalid search. */ + private static OpenSearchResponse empty() { + return new OpenSearchResponse(SearchHits.empty(), null, List.of(), false); + } + + @TestOnly + public static OpenSearchResponse of( SearchResponse searchResponse, OpenSearchExprValueFactory exprValueFactory, List includes) { - this(searchResponse, exprValueFactory, includes, false); + return new OpenSearchResponse(searchResponse, exprValueFactory, includes, false); } /** Constructor of OpenSearchResponse. */ @@ -94,22 +104,43 @@ public OpenSearchResponse( * @return true for empty */ public boolean isEmpty() { - return (hits.getHits() == null) - || (((hits.getHits().length == 0) && aggregations == null) - && (!isCountAgg - || hits.getTotalHits() == null)); // check total hits if is count aggregation + if (isCountResponse()) { + return hits.getTotalHits() == null; + } else if (isAggregationResponse()) { + return aggregations.asList().isEmpty(); + } else { + return getHitsSize() == 0; + } } public boolean isAggregationResponse() { return aggregations != null; } + public boolean isCompositeAggregationResponse() { + return isAggregationResponse() + && !aggregations.asList().isEmpty() + && (aggregations.asList().get(0) instanceof InternalComposite); + } + + /** + * Get the size of composite aggregation bucket. Must be called after + * isCompositeAggregationResponse() is true. + */ + public int getCompositeBucketSize() { + if (isCompositeAggregationResponse()) { + return ((InternalComposite) aggregations.asList().get(0)).getBuckets().size(); + } + assert false : "Should never call here"; + return -1; + } + public boolean isCountResponse() { return isCountAgg; } public int getHitsSize() { - return hits.getHits() == null ? 0 : hits.getHits().length; + return hits == null ? 0 : hits.getHits() == null ? 0 : hits.getHits().length; } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java index ab9ec64d102..d7539312cd1 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/OpenSearchIndex.java @@ -5,6 +5,8 @@ package org.opensearch.sql.opensearch.storage; +import static org.opensearch.search.aggregations.MultiBucketConsumerService.DEFAULT_MAX_BUCKETS; + import com.google.common.annotations.VisibleForTesting; import java.util.HashMap; import java.util.LinkedHashMap; @@ -198,10 +200,16 @@ public Integer getMaxResultWindow() { return cachedMaxResultWindow; } - public Integer getBucketSize() { - return Math.min( - settings.getSettingValue(Settings.Key.QUERY_BUCKET_SIZE), - settings.getSettingValue(Settings.Key.SEARCH_MAX_BUCKETS)); + public Integer getQueryBucketSize() { + return Math.min(settings.getSettingValue(Settings.Key.QUERY_BUCKET_SIZE), getMaxBuckets()); + } + + public Integer getMaxBuckets() { + try { + return settings.getSettingValue(Settings.Key.SEARCH_MAX_BUCKETS); + } catch (Exception e) { + return DEFAULT_MAX_BUCKETS; + } } /** TODO: Push down operations to index scan operator as much as possible in future. */ @@ -293,6 +301,7 @@ public OpenSearchResourceMonitor createOpenSearchResourceMonitor() { return new OpenSearchResourceMonitor(getSettings(), new OpenSearchMemoryHealthy(settings)); } + /** The v3 API to build an OpenSearchRequest, calling by CalciteEnumerableIndexScan */ public OpenSearchRequest buildRequest(OpenSearchRequestBuilder requestBuilder) { final TimeValue cursorKeepAlive = settings.getSettingValue(Settings.Key.SQL_CURSOR_KEEP_ALIVE); return requestBuilder.build( diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 80ad06422e2..a04897ce9b4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -9,7 +9,6 @@ import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_PUSHDOWN_ROWCOUNT_ESTIMATION_FACTOR; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -125,7 +124,7 @@ public double estimateRowCount(RelMetadataQuery mq) { case AGGREGATION -> mq.getRowCount((RelNode) operation.digest()); case PROJECT, SORT, SORT_EXPR -> rowCount; case SORT_AGG_METRICS -> - NumberUtil.min(rowCount, osIndex.getBucketSize().doubleValue()); + NumberUtil.min(rowCount, osIndex.getQueryBucketSize().doubleValue()); // Refer the org.apache.calcite.rel.metadata.RelMdRowCount case FILTER, SCRIPT -> NumberUtil.multiply( @@ -265,35 +264,6 @@ protected List getCollationNames(List collations) { .toList(); } - /** - * Check if all sort-by collations equal aggregators that are pushed down. E.g. In `stats avg(age) - * as avg_age, sum(age) as sum_age by state | sort avg_age, sum_age`, the sort keys `avg_age`, - * `sum_age` which equal the pushed down aggregators `avg(age)`, `sum(age)`. - * - * @param collations List of collation names to check against aggregators. - * @return True if all collation names match all aggregator output, false otherwise. - */ - protected boolean isAllCollationNamesEqualAggregators(List collations) { - Stream aggregates = - pushDownContext.stream() - .filter(action -> action.type() == PushDownType.AGGREGATION) - .map(action -> ((LogicalAggregate) action.digest())); - return aggregates - .map(aggregate -> isAllCollationNamesEqualAggregators(aggregate, collations)) - .reduce(false, Boolean::logicalOr); - } - - private boolean isAllCollationNamesEqualAggregators( - LogicalAggregate aggregate, List collations) { - List fieldNames = aggregate.getRowType().getFieldNames(); - // The output fields of the aggregate are in the format of - // [...grouping fields, ...aggregator fields], so we set an offset to skip - // the grouping fields. - int groupOffset = aggregate.getGroupSet().cardinality(); - List fieldsWithoutGrouping = fieldNames.subList(groupOffset, fieldNames.size()); - return new HashSet<>(collations).equals(new HashSet<>(fieldsWithoutGrouping)); - } - /** * Check if any sort-by collations is in aggregators that are pushed down. E.g. In `stats avg(age) * as avg_age by state | sort avg_age`, the sort clause has `avg_age` which is an aggregator. The diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java index 4019346e055..a0b33f3b541 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScanner.java @@ -13,6 +13,8 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executor; import javax.annotation.Nullable; +import org.opensearch.OpenSearchException; +import org.opensearch.OpenSearchSecurityException; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.exception.NonFallbackCalciteException; import org.opensearch.sql.opensearch.client.OpenSearchClient; @@ -63,9 +65,14 @@ public class BackgroundSearchScanner { @Nullable private final Executor backgroundExecutor; private CompletableFuture nextBatchFuture = null; private boolean stopIteration = false; + private final int maxResultWindow; + private final int queryBucketSize; - public BackgroundSearchScanner(OpenSearchClient client) { + public BackgroundSearchScanner( + OpenSearchClient client, int maxResultWindow, int queryBucketSize) { this.client = client; + this.maxResultWindow = maxResultWindow; + this.queryBucketSize = queryBucketSize; // We can only actually do the background operation if we have the ability to access the thread // pool. Otherwise, fallback to synchronous fetch. if (client.getNodeClient().isPresent()) { @@ -104,11 +111,26 @@ private OpenSearchResponse getCurrentResponse(OpenSearchRequest request) { if (isAsync()) { try { return nextBatchFuture.get(); + } catch (OpenSearchSecurityException e) { + throw e; } catch (InterruptedException | ExecutionException e) { + if (e.getCause() instanceof OpenSearchSecurityException) { + throw (OpenSearchSecurityException) e.getCause(); + } + if (e.getCause() instanceof OpenSearchException) { + if (((OpenSearchException) e.getCause()).getRootCause() + instanceof ArrayIndexOutOfBoundsException) { + // It could cause by searching CompositeAggregator with the last afterKey + // which is not exist in the index. + // In this case, we can safely ignore this exception. + return OpenSearchResponse.EMPTY; + } + } throw new NonFallbackCalciteException( "Failed to fetch data from the index: the background task failed or interrupted.\n" + " Inner error: " - + e.getMessage()); + + e.getMessage(), + e); } } else { return client.search(request); @@ -120,18 +142,23 @@ private OpenSearchResponse getCurrentResponse(OpenSearchRequest request) { * also trigger the next background fetch. * * @param request The OpenSearch request to execute - * @param maxResultWindow Maximum number of results to fetch per batch * @return SearchBatchResult containing the current batch's iterator and completion status * @throws NonFallbackCalciteException if the background fetch fails or is interrupted */ - public SearchBatchResult fetchNextBatch(OpenSearchRequest request, int maxResultWindow) { + public SearchBatchResult fetchNextBatch(OpenSearchRequest request) { OpenSearchResponse response = getCurrentResponse(request); // Determine if we need future batches - if (response.isAggregationResponse() - || response.isCountResponse() - || response.getHitsSize() < maxResultWindow) { + if (response.isCountResponse()) { stopIteration = true; + } else if (response.isCompositeAggregationResponse()) { + // For composite aggregations, if we get fewer buckets than requested, we're done + stopIteration = response.getCompositeBucketSize() < queryBucketSize; + } else if (response.isAggregationResponse()) { + stopIteration = true; + } else { + // For regular search results, if we get fewer hits than maxResultWindow, we're done + stopIteration = response.getHitsSize() < maxResultWindow; } Iterator iterator; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java index ddf14cfee38..b00b00a8f17 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java @@ -118,6 +118,7 @@ public Enumerator enumerator() { getRowType().getFieldNames(), requestBuilder.getMaxResponseSize(), requestBuilder.getMaxResultWindow(), + osIndex.getQueryBucketSize(), osIndex.buildRequest(requestBuilder), osIndex.createOpenSearchResourceMonitor()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index 4aaac41cc75..cd9d42b8955 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -315,13 +315,15 @@ public CalciteLogicalIndexScan pushDownSortAggregateMeasure(Sort sort) { return null; } CalciteLogicalIndexScan newScan = copyWithNewTraitSet(sort.getTraitSet()); - AbstractAction newAction = - (AggregationBuilderAction) - aggAction -> - aggAction.rePushDownSortAggMeasure( - sort.getCollation().getFieldCollations(), rowType.getFieldNames()); + newScan + .pushDownContext + .getAggPushDownAction() + .rePushDownSortAggMeasure( + sort.getCollation().getFieldCollations(), rowType.getFieldNames()); + AbstractAction action = + (OSRequestBuilderAction) requestAction -> requestAction.resetRequestTotal(); Object digest = sort.getCollation().getFieldCollations(); - newScan.pushDownContext.add(PushDownType.SORT_AGG_METRICS, digest, newAction); + newScan.pushDownContext.add(PushDownType.SORT_AGG_METRICS, digest, action); return newScan; } catch (Exception e) { if (LOG.isDebugEnabled()) { @@ -334,9 +336,10 @@ public CalciteLogicalIndexScan pushDownSortAggregateMeasure(Sort sort) { public CalciteLogicalIndexScan pushDownRareTop(Project project, RareTopDigest digest) { try { CalciteLogicalIndexScan newScan = copyWithNewSchema(project.getRowType()); - AbstractAction newAction = - (AggregationBuilderAction) aggAction -> aggAction.rePushDownRareTop(digest); - newScan.pushDownContext.add(PushDownType.RARE_TOP, digest, newAction); + newScan.pushDownContext.getAggPushDownAction().rePushDownRareTop(digest); + AbstractAction action = + (OSRequestBuilderAction) requestAction -> requestAction.resetRequestTotal(); + newScan.pushDownContext.add(PushDownType.RARE_TOP, digest, action); return newScan; } catch (Exception e) { if (LOG.isDebugEnabled()) { @@ -364,7 +367,7 @@ public AbstractRelNode pushDownAggregate(Aggregate aggregate, Project project) { .filter(entry -> schema.contains(entry.getKey())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); List outputFields = aggregate.getRowType().getFieldNames(); - int bucketSize = osIndex.getBucketSize(); + int queryBucketSize = osIndex.getQueryBucketSize(); boolean bucketNullable = Boolean.parseBoolean( aggregate.getHints().stream() @@ -374,7 +377,7 @@ public AbstractRelNode pushDownAggregate(Aggregate aggregate, Project project) { .orElseGet(() -> "true")); AggregateAnalyzer.AggregateBuilderHelper helper = new AggregateAnalyzer.AggregateBuilderHelper( - getRowType(), fieldTypes, getCluster(), bucketNullable, bucketSize); + getRowType(), fieldTypes, getCluster(), bucketNullable, queryBucketSize); final Pair, OpenSearchAggregationResponseParser> builderAndParser = AggregateAnalyzer.analyze(aggregate, project, outputFields, helper); Map extendedTypeMapping = @@ -406,17 +409,24 @@ public AbstractRelNode pushDownLimit(LogicalSort sort, Integer limit, Integer of if (pushDownContext.isAggregatePushed()) { // Push down the limit into the aggregation bucket in advance to detect whether the limit // can update the aggregation builder - boolean updated = + boolean canUpdate = pushDownContext.getAggPushDownAction().pushDownLimitIntoBucketSize(limit + offset); - if (!updated && offset > 0) return null; + if (!canUpdate && offset > 0) return null; CalciteLogicalIndexScan newScan = this.copyWithNewSchema(getRowType()); - // Simplify the action if it doesn't update the aggregation builder, otherwise keep the - // original action - // It won't change the aggregation builder by do this action again since it's idempotent - AggregationBuilderAction action = - updated - ? aggAction -> aggAction.pushDownLimitIntoBucketSize(limit + offset) - : aggAction -> {}; + if (canUpdate) { + newScan + .pushDownContext + .getAggPushDownAction() + .pushDownLimitIntoBucketSize(limit + offset); + } + AbstractAction action; + if (pushDownContext.getAggPushDownAction().isCompositeAggregation()) { + action = + (OSRequestBuilderAction) + requestBuilder -> requestBuilder.pushDownLimitToRequestTotal(limit, offset); + } else { + action = (AggregationBuilderAction) aggAction -> {}; + } newScan.pushDownContext.add(PushDownType.LIMIT, new LimitDigest(limit, offset), action); return offset > 0 ? sort.copy(sort.getTraitSet(), List.of(newScan)) : newScan; } else { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java index e684d128914..05bd00dcf2c 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java @@ -40,9 +40,6 @@ public class OpenSearchIndexEnumerator implements Enumerator { /** Largest number of rows allowed in the response. */ @EqualsAndHashCode.Include @ToString.Include private final int maxResponseSize; - /** Largest number of rows allowed in the response. */ - @EqualsAndHashCode.Include @ToString.Include private final int maxResultWindow; - /** How many moveNext() calls to perform resource check once. */ private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; @@ -62,6 +59,7 @@ public OpenSearchIndexEnumerator( List fields, int maxResponseSize, int maxResultWindow, + int queryBucketSize, OpenSearchRequest request, ResourceMonitor monitor) { if (!monitor.isHealthy()) { @@ -71,16 +69,14 @@ public OpenSearchIndexEnumerator( this.fields = fields; this.request = request; this.maxResponseSize = maxResponseSize; - this.maxResultWindow = maxResultWindow; this.monitor = monitor; this.client = client; - this.bgScanner = new BackgroundSearchScanner(client); + this.bgScanner = new BackgroundSearchScanner(client, maxResultWindow, queryBucketSize); this.bgScanner.startScanning(request); } private Iterator fetchNextBatch() { - BackgroundSearchScanner.SearchBatchResult result = - bgScanner.fetchNextBatch(request, maxResultWindow); + BackgroundSearchScanner.SearchBatchResult result = bgScanner.fetchNextBatch(request); return result.iterator(); } @@ -125,7 +121,7 @@ public boolean moveNext() { @Override public void reset() { bgScanner.reset(request); - iterator = bgScanner.fetchNextBatch(request, maxResultWindow).iterator(); + iterator = bgScanner.fetchNextBatch(request).iterator(); queryCount = 0; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggPushDownAction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggPushDownAction.java index 5d96caff9f9..81595cc56fd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggPushDownAction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggPushDownAction.java @@ -5,6 +5,8 @@ package org.opensearch.sql.opensearch.storage.scan.context; +import static org.opensearch.search.aggregations.MultiBucketConsumerService.DEFAULT_MAX_BUCKETS; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -43,8 +45,6 @@ @Getter @EqualsAndHashCode public class AggPushDownAction implements OSRequestBuilderAction { - private static final int MAX_BUCKET_SIZE = 65535; - private Pair, OpenSearchAggregationResponseParser> builderAndParser; private final Map extendedTypeMapping; private final long scriptCount; @@ -186,13 +186,13 @@ public void rePushDownRareTop(RareTopDigest digest) { for (int i = 0; i < composite.sources().size(); i++) { TermsValuesSourceBuilder terms = (TermsValuesSourceBuilder) composite.sources().get(i); if (i == 0) { // first - aggregationBuilder = buildTermsAggregationBuilder(terms, null, MAX_BUCKET_SIZE); + aggregationBuilder = buildTermsAggregationBuilder(terms, null, DEFAULT_MAX_BUCKETS); } else if (i == composite.sources().size() - 1) { // last aggregationBuilder.subAggregation( buildTermsAggregationBuilder(terms, bucketOrder, digest.number())); } else { aggregationBuilder.subAggregation( - buildTermsAggregationBuilder(terms, null, MAX_BUCKET_SIZE)); + buildTermsAggregationBuilder(terms, null, DEFAULT_MAX_BUCKETS)); } } } else { @@ -390,6 +390,11 @@ public void pushDownSortIntoAggBucket( // TODO for MultiTermsAggregationBuilder } + public boolean isCompositeAggregation() { + return builderAndParser.getLeft().stream() + .anyMatch(builder -> builder instanceof CompositeAggregationBuilder); + } + /** * Check if the limit can be pushed down into aggregation bucket when the limit size is less than * bucket number. diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggregationBuilderAction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggregationBuilderAction.java index f2362e3c1cc..f9f43c89a7b 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggregationBuilderAction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/AggregationBuilderAction.java @@ -11,6 +11,6 @@ default void pushOperation(PushDownContext context, PushDownOperation operation) // Apply transformation to aggregation builder in the optimization phase as some transformation // may cause exception. We need to detect that exception in advance. apply(context.getAggPushDownAction()); - context.getOperationsForAgg().add(operation); + context.addOperationForAgg(operation); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/OSRequestBuilderAction.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/OSRequestBuilderAction.java index 6abe97266f7..d3a762c22d8 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/OSRequestBuilderAction.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/OSRequestBuilderAction.java @@ -10,6 +10,6 @@ /** A lambda action to apply on the {@link OpenSearchRequestBuilder} */ public interface OSRequestBuilderAction extends AbstractAction { default void pushOperation(PushDownContext context, PushDownOperation operation) { - context.getOperationsForRequestBuilder().add(operation); + context.addOperationForRequestBuilder(operation); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownContext.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownContext.java index 16c81facace..4f84746ade4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownContext.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/context/PushDownContext.java @@ -5,10 +5,8 @@ package org.opensearch.sql.opensearch.storage.scan.context; -import com.google.common.collect.Iterators; import java.util.AbstractCollection; import java.util.ArrayDeque; -import java.util.Collections; import java.util.Iterator; import lombok.Getter; import org.jetbrains.annotations.NotNull; @@ -20,6 +18,7 @@ @Getter public class PushDownContext extends AbstractCollection { private final OpenSearchIndex osIndex; + private ArrayDeque queue = new ArrayDeque<>(); private ArrayDeque operationsForRequestBuilder; private boolean isAggregatePushed = false; @@ -67,33 +66,28 @@ public PushDownContext cloneWithoutSort() { @NotNull @Override public Iterator iterator() { - if (operationsForRequestBuilder == null) { - return Collections.emptyIterator(); - } else if (operationsForAgg == null) { - return operationsForRequestBuilder.iterator(); - } else { - return Iterators.concat(operationsForRequestBuilder.iterator(), operationsForAgg.iterator()); - } + return queue.iterator(); } @Override public int size() { - return (operationsForRequestBuilder == null ? 0 : operationsForRequestBuilder.size()) - + (operationsForAgg == null ? 0 : operationsForAgg.size()); + return queue.size(); } - ArrayDeque getOperationsForRequestBuilder() { + void addOperationForRequestBuilder(PushDownOperation operation) { if (operationsForRequestBuilder == null) { this.operationsForRequestBuilder = new ArrayDeque<>(); } - return operationsForRequestBuilder; + operationsForRequestBuilder.add(operation); + queue.add(operation); } - ArrayDeque getOperationsForAgg() { + void addOperationForAgg(PushDownOperation operation) { if (operationsForAgg == null) { this.operationsForAgg = new ArrayDeque<>(); } - return operationsForAgg; + operationsForAgg.add(operation); + queue.add(operation); } @Override diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index 9b6da17567e..81261aa7a70 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -409,7 +409,7 @@ void cleanup_rethrows_exception() { @SneakyThrows void cleanup_pit_request() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), new SearchSourceBuilder(), factory, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index f93da87c303..afd210da1ff 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -435,7 +435,7 @@ void cleanup_with_IOException() { @SneakyThrows void cleanup_pit_request() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), new SearchSourceBuilder(), factory, @@ -453,7 +453,7 @@ void cleanup_pit_request() { void cleanup_pit_request_throw_exception() { when(restClient.deletePit(any(), any())).thenThrow(new IOException()); OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), new SearchSourceBuilder(), factory, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java index 5ce83c66ff8..5fee9503ca6 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequestTest.java @@ -60,10 +60,10 @@ public class OpenSearchQueryRequestTest { @Mock private OpenSearchExprValueFactory factory; private final OpenSearchQueryRequest request = - new OpenSearchQueryRequest("test", 200, factory, List.of()); + OpenSearchQueryRequest.of("test", 200, factory, List.of()); private final OpenSearchQueryRequest remoteRequest = - new OpenSearchQueryRequest("ccs:test", 200, factory, List.of()); + OpenSearchQueryRequest.of("ccs:test", 200, factory, List.of()); @Mock private StreamOutput streamOutput; @Mock private StreamInput streamInput; @@ -88,7 +88,7 @@ void testWriteTo() throws IOException { sourceBuilderForSerializer.searchAfter(new Object[] {"value1", 123}); List includes = List.of("field1", "field2"); serializationRequest = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilderForSerializer, factory, @@ -120,7 +120,7 @@ void testWriteToWithoutSearchAfter() List includes = List.of("field1", "field2"); serializationRequest = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilderForSerializer, factory, @@ -139,7 +139,7 @@ void testWriteToWithoutSearchAfter() @Test void testWriteToWithoutPIT() { - serializationRequest = new OpenSearchQueryRequest("test", 200, factory, List.of()); + serializationRequest = OpenSearchQueryRequest.of("test", 200, factory, List.of()); UnsupportedOperationException exception = assertThrows( @@ -155,24 +155,21 @@ void testWriteToWithoutPIT() { @Test void search() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( - new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, List.of()); + OpenSearchQueryRequest.of("test", sourceBuilder, factory, List.of()); when(searchAction.apply(any())).thenReturn(searchResponse); when(searchResponse.getHits()).thenReturn(searchHits); when(searchHits.getHits()).thenReturn(new SearchHit[] {searchHit}); - OpenSearchResponse searchResponse = request.search(searchAction, scrollAction); - assertFalse(searchResponse.isEmpty()); - searchResponse = request.search(searchAction, scrollAction); - assertTrue(searchResponse.isEmpty()); + OpenSearchResponse response = request.search(searchAction, scrollAction); + assertFalse(response.isEmpty()); verify(searchAction, times(1)).apply(any()); } @Test void search_with_pit() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, @@ -203,7 +200,7 @@ void search_with_pit() { @Test void search_with_pit_hits_null() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, @@ -224,7 +221,7 @@ void search_with_pit_hits_empty() { SearchResponse searchResponse = mock(SearchResponse.class); SearchHits searchHits = mock(SearchHits.class); OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, @@ -245,7 +242,7 @@ void search_with_pit_null() { SearchResponse searchResponse = mock(SearchResponse.class); SearchHits searchHits = mock(SearchHits.class); OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, @@ -264,7 +261,7 @@ void search_with_pit_null() { @Test void has_another_batch() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, @@ -277,7 +274,7 @@ void has_another_batch() { @Test void has_another_batch_pid_null() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, @@ -290,7 +287,7 @@ void has_another_batch_pid_null() { @Test void has_another_batch_need_clean() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, @@ -308,8 +305,7 @@ void has_another_batch_need_clean() { @Test void search_withoutContext() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( - new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, List.of()); + OpenSearchQueryRequest.of("test", sourceBuilder, factory, List.of()); when(searchAction.apply(any())).thenReturn(searchResponse); when(searchResponse.getHits()).thenReturn(searchHits); @@ -322,8 +318,7 @@ void search_withoutContext() { @Test void search_withIncludes() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( - new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, List.of()); + OpenSearchQueryRequest.of("test", sourceBuilder, factory, List.of()); String[] includes = {"_id", "_index"}; when(searchAction.apply(any())).thenReturn(searchResponse); @@ -333,9 +328,6 @@ void search_withIncludes() { OpenSearchResponse searchResponse = request.search(searchAction, scrollAction); assertFalse(searchResponse.isEmpty()); - searchResponse = request.search(searchAction, scrollAction); - assertTrue(searchResponse.isEmpty()); - verify(searchAction, times(1)).apply(any()); } @@ -348,7 +340,7 @@ void clean() { @Test void testCleanConditionTrue() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, @@ -371,7 +363,7 @@ void testCleanConditionTrue() { @Test void testCleanConditionFalse_needCleanFalse() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, @@ -393,7 +385,7 @@ void testCleanConditionFalse_needCleanFalse() { @Test void testCleanConditionFalse_pidNull() { OpenSearchQueryRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), sourceBuilder, factory, diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java index 8db9176fb5e..1f21f1e769e 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java @@ -96,8 +96,8 @@ void build_query_request() { requestBuilder.pushDownTrackedScore(true); assertEquals( - new OpenSearchQueryRequest( - new OpenSearchRequest.IndexName("test"), + OpenSearchQueryRequest.of( + "test", new SearchSourceBuilder() .from(offset) .size(limit) @@ -105,7 +105,7 @@ void build_query_request() { .trackScores(true), exprValueFactory, List.of()), - requestBuilder.build(indexName, DEFAULT_QUERY_TIMEOUT, client)); + requestBuilder.build(indexName, null, client)); } @Test @@ -127,7 +127,7 @@ void build_PIT_request_with_correct_size() { requestBuilder.pushDownPageSize(2); assertEquals( - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), new SearchSourceBuilder().from(offset).size(2).timeout(DEFAULT_QUERY_TIMEOUT), exprValueFactory, @@ -146,7 +146,7 @@ void buildRequestWithPit_pageSizeNull_sizeGreaterThanMaxResultWindow() { requestBuilder.pushDownLimit(limit, offset); assertEquals( - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), new SearchSourceBuilder() .from(offset) @@ -167,12 +167,12 @@ void buildRequestWithPit_pageSizeNull_sizeLessThanMaxResultWindow() { requestBuilder.pushDownLimit(limit, offset); assertEquals( - new OpenSearchQueryRequest( - new OpenSearchRequest.IndexName("test"), + OpenSearchQueryRequest.of( + "test", new SearchSourceBuilder().from(offset).size(limit).timeout(DEFAULT_QUERY_TIMEOUT), exprValueFactory, List.of()), - requestBuilder.build(indexName, DEFAULT_QUERY_TIMEOUT, client)); + requestBuilder.build(indexName, null, client)); } @Test @@ -185,7 +185,7 @@ void buildRequestWithPit_pageSizeNotNull_startFromZero() { when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId"); assertEquals( - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), new SearchSourceBuilder().from(offset).size(pageSize).timeout(DEFAULT_QUERY_TIMEOUT), exprValueFactory, @@ -434,7 +434,7 @@ void test_push_down_project() { requestBuilder); assertEquals( - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), new SearchSourceBuilder() .from(DEFAULT_OFFSET) @@ -469,8 +469,8 @@ void test_push_down_project_limit() { requestBuilder); assertEquals( - new OpenSearchQueryRequest( - new OpenSearchRequest.IndexName("test"), + OpenSearchQueryRequest.of( + "test", new SearchSourceBuilder() .from(offset) .size(limit) @@ -478,7 +478,7 @@ void test_push_down_project_limit() { .fetchSource("intA", null), exprValueFactory, List.of("intA")), - requestBuilder.build(indexName, DEFAULT_QUERY_TIMEOUT, client)); + requestBuilder.build(indexName, null, client)); } @Test @@ -499,8 +499,8 @@ void test_push_down_project_limit_and_offset() { requestBuilder); assertEquals( - new OpenSearchQueryRequest( - new OpenSearchRequest.IndexName("test"), + OpenSearchQueryRequest.of( + "test", new SearchSourceBuilder() .from(offset) .size(limit) @@ -508,7 +508,7 @@ void test_push_down_project_limit_and_offset() { .fetchSource("intA", null), exprValueFactory, List.of("intA")), - requestBuilder.build(indexName, DEFAULT_QUERY_TIMEOUT, client)); + requestBuilder.build(indexName, null, client)); } @Test @@ -562,7 +562,7 @@ void test_push_down_project_with_alias_type() { requestBuilder); assertEquals( - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( new OpenSearchRequest.IndexName("test"), new SearchSourceBuilder() .from(DEFAULT_OFFSET) diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java index 6c509c20c36..f9897f48dd2 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java @@ -5,7 +5,6 @@ package org.opensearch.sql.opensearch.response; -import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -35,6 +34,7 @@ import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.SearchShardTarget; +import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.fetch.subphase.highlight.HighlightField; import org.opensearch.sql.data.model.ExprFloatValue; @@ -79,24 +79,25 @@ void isEmpty() { new TotalHits(2L, TotalHits.Relation.EQUAL_TO), 1.0F)); - var response = new OpenSearchResponse(searchResponse, factory, includes); + var response = OpenSearchResponse.of(searchResponse, factory, includes); assertFalse(response.isEmpty()); when(searchResponse.getHits()).thenReturn(SearchHits.empty()); when(searchResponse.getAggregations()).thenReturn(null); - response = new OpenSearchResponse(searchResponse, factory, includes); + response = OpenSearchResponse.of(searchResponse, factory, includes); assertTrue(response.isEmpty()); when(searchResponse.getHits()) .thenReturn(new SearchHits(null, new TotalHits(0, TotalHits.Relation.EQUAL_TO), 0)); - response = new OpenSearchResponse(searchResponse, factory, includes); + response = OpenSearchResponse.of(searchResponse, factory, includes); assertTrue(response.isEmpty()); when(searchResponse.getHits()).thenReturn(SearchHits.empty()); - when(searchResponse.getAggregations()).thenReturn(new Aggregations(emptyList())); + when(searchResponse.getAggregations()) + .thenReturn(new Aggregations(List.of(mock(Aggregation.class)))); - response = new OpenSearchResponse(searchResponse, factory, includes); + response = OpenSearchResponse.of(searchResponse, factory, includes); assertFalse(response.isEmpty()); } @@ -118,7 +119,7 @@ void iterator() { .thenReturn(exprTupleValue2); int i = 0; - for (ExprValue hit : new OpenSearchResponse(searchResponse, factory, List.of("id1"))) { + for (ExprValue hit : OpenSearchResponse.of(searchResponse, factory, List.of("id1"))) { if (i == 0) { assertEquals(exprTupleValue1.tupleValue().get("id"), hit.tupleValue().get("id")); } else if (i == 1) { @@ -168,7 +169,7 @@ void iterator_metafields() { List includes = List.of("id1", "_index", "_id", "_routing", "_sort", "_score", "_maxscore"); int i = 0; - for (ExprValue hit : new OpenSearchResponse(searchResponse, factory, includes)) { + for (ExprValue hit : OpenSearchResponse.of(searchResponse, factory, includes)) { if (i == 0) { assertEquals(exprTupleResponse, hit); } else { @@ -199,7 +200,7 @@ void iterator_metafields_withoutIncludes() { ExprTupleValue exprTupleResponse = ExprTupleValue.fromExprValueMap(ImmutableMap.of("id1", new ExprIntegerValue(1))); int i = 0; - for (ExprValue hit : new OpenSearchResponse(searchResponse, factory, includes)) { + for (ExprValue hit : OpenSearchResponse.of(searchResponse, factory, includes)) { if (i == 0) { assertEquals(exprTupleResponse, hit); } else { @@ -239,7 +240,7 @@ void iterator_metafields_scoreNaN() { "_id", new ExprStringValue("testId"), "_sort", new ExprLongValue(123456L))); int i = 0; - for (ExprValue hit : new OpenSearchResponse(searchResponse, factory, includes)) { + for (ExprValue hit : OpenSearchResponse.of(searchResponse, factory, includes)) { if (i == 0) { assertEquals(exprTupleResponse, hit); } else { @@ -263,7 +264,7 @@ void iterator_with_inner_hits() { when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleValue1); - for (ExprValue hit : new OpenSearchResponse(searchResponse, factory, List.of("id1"))) { + for (ExprValue hit : OpenSearchResponse.of(searchResponse, factory, List.of("id1"))) { assertEquals(exprTupleValue1, hit); } } @@ -272,7 +273,7 @@ void iterator_with_inner_hits() { void response_is_aggregation_when_aggregation_not_empty() { when(searchResponse.getAggregations()).thenReturn(aggregations); - OpenSearchResponse response = new OpenSearchResponse(searchResponse, factory, includes); + OpenSearchResponse response = OpenSearchResponse.of(searchResponse, factory, includes); assertTrue(response.isAggregationResponse()); } @@ -280,7 +281,7 @@ void response_is_aggregation_when_aggregation_not_empty() { void response_isnot_aggregation_when_aggregation_is_empty() { when(searchResponse.getAggregations()).thenReturn(null); - OpenSearchResponse response = new OpenSearchResponse(searchResponse, factory, includes); + OpenSearchResponse response = OpenSearchResponse.of(searchResponse, factory, includes); assertFalse(response.isAggregationResponse()); } @@ -297,7 +298,7 @@ void aggregation_iterator() { .thenReturn(new ExprIntegerValue(2)); int i = 0; - for (ExprValue hit : new OpenSearchResponse(searchResponse, factory, includes)) { + for (ExprValue hit : OpenSearchResponse.of(searchResponse, factory, includes)) { if (i == 0) { assertEquals(exprTupleValue1, hit); } else if (i == 1) { @@ -329,7 +330,7 @@ void highlight_iterator() { when(searchHit1.getHighlightFields()).thenReturn(highlightMap); when(factory.construct(any(), anyBoolean())).thenReturn(resultTuple); - for (ExprValue resultHit : new OpenSearchResponse(searchResponse, factory, includes)) { + for (ExprValue resultHit : OpenSearchResponse.of(searchResponse, factory, includes)) { var expected = ExprValueUtils.collectionValue( Arrays.stream(searchHit.getHighlightFields().get("highlights").getFragments()) diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScannerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScannerTest.java index f4a7f297df9..c7531c37758 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScannerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/BackgroundSearchScannerTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Test; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.opensearch.client.OpenSearchClient; +import org.opensearch.sql.opensearch.request.OpenSearchQueryRequest; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.response.OpenSearchResponse; import org.opensearch.threadpool.ThreadPool; @@ -40,14 +41,14 @@ void setUp() { client = mock(OpenSearchClient.class); nodeClient = mock(NodeClient.class); threadPool = mock(ThreadPool.class); - request = mock(OpenSearchRequest.class); + request = mock(OpenSearchQueryRequest.class); executor = Executors.newSingleThreadExecutor(); when(client.getNodeClient()).thenReturn(Optional.of(nodeClient)); when(nodeClient.threadPool()).thenReturn(threadPool); when(threadPool.executor(any())).thenReturn(executor); - scanner = new BackgroundSearchScanner(client); + scanner = new BackgroundSearchScanner(client, 10, 10); } @Test @@ -55,13 +56,13 @@ void testSyncFallbackWhenNoNodeClient() { // Setup client without node client OpenSearchClient syncClient = mock(OpenSearchClient.class); when(syncClient.getNodeClient()).thenReturn(Optional.empty()); - scanner = new BackgroundSearchScanner(syncClient); + scanner = new BackgroundSearchScanner(syncClient, 10, 10); OpenSearchResponse response = mockResponse(false, false, 10); when(syncClient.search(request)).thenReturn(response); scanner.startScanning(request); - BackgroundSearchScanner.SearchBatchResult result = scanner.fetchNextBatch(request, 10); + BackgroundSearchScanner.SearchBatchResult result = scanner.fetchNextBatch(request); assertFalse( result.stopIteration(), "Expected iteration to continue after fetching one full page"); @@ -80,13 +81,13 @@ void testCompleteScanWithMultipleBatches() { scanner.startScanning(request); // First batch - BackgroundSearchScanner.SearchBatchResult result1 = scanner.fetchNextBatch(request, 10); + BackgroundSearchScanner.SearchBatchResult result1 = scanner.fetchNextBatch(request); assertFalse( result1.stopIteration(), "Expected iteration to continue after fetching 10/15 results"); assertTrue(result1.iterator().hasNext()); // Second batch - BackgroundSearchScanner.SearchBatchResult result2 = scanner.fetchNextBatch(request, 10); + BackgroundSearchScanner.SearchBatchResult result2 = scanner.fetchNextBatch(request); assertTrue(result2.stopIteration()); assertFalse(result2.iterator().hasNext()); } @@ -97,7 +98,7 @@ void testFetchOnceForAggregationResponse() { when(client.search(request)).thenReturn(response); scanner.startScanning(request); - BackgroundSearchScanner.SearchBatchResult result = scanner.fetchNextBatch(request, 10); + BackgroundSearchScanner.SearchBatchResult result = scanner.fetchNextBatch(request); assertTrue(scanner.isScanDone()); } @@ -108,7 +109,7 @@ void testFetchOnceWhenResultsBelowWindow() { when(client.search(request)).thenReturn(response); scanner.startScanning(request); - BackgroundSearchScanner.SearchBatchResult result = scanner.fetchNextBatch(request, 10); + BackgroundSearchScanner.SearchBatchResult result = scanner.fetchNextBatch(request); assertTrue(scanner.isScanDone()); } @@ -121,8 +122,8 @@ void testReset() { when(client.search(request)).thenReturn(response1).thenReturn(response2); scanner.startScanning(request); - scanner.fetchNextBatch(request, 10); - scanner.fetchNextBatch(request, 10); + scanner.fetchNextBatch(request); + scanner.fetchNextBatch(request); assertTrue(scanner.isScanDone()); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java index 05d0c78ce31..fb511645d62 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanPaginationTest.java @@ -95,7 +95,7 @@ void dont_serialize_if_no_cursor() { OpenSearchRequest request = mock(); OpenSearchResponse response = mock(); when(builder.build(any(), any(), any())).thenReturn(request); - when(client.search(any())).thenReturn(response); + lenient().when(client.search(any())).thenReturn(response); try (var indexScan = new OpenSearchIndexScan(client, builder.build(INDEX_NAME, SCROLL_TIMEOUT, client))) { indexScan.open(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java index 08d60b66a3a..88bc6e1190d 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanTest.java @@ -119,11 +119,11 @@ void serialize_PIT(Integer numberOfIncludes) { .map(i -> "column" + i) .collect(Collectors.toList()); var request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( INDEX_NAME, searchSourceBuilder, factory, includes, CURSOR_KEEP_ALIVE, "samplePitId"); // make a response, so OpenSearchResponse::isEmpty would return true and unset needClean var response = mock(SearchResponse.class); - when(response.getAggregations()).thenReturn(mock()); + when(response.getAggregations()).thenReturn(null); var hits = mock(SearchHits.class); when(response.getHits()).thenReturn(hits); SearchHit hit = mock(SearchHit.class); @@ -396,7 +396,7 @@ PushDownAssertion shouldQueryHighlight(QueryBuilder query, HighlightBuilder high .size(MAX_RESULT_WINDOW) .highlighter(highlight); OpenSearchRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( EMPLOYEES_INDEX, sourceBuilder, factory, List.of(), CURSOR_KEEP_ALIVE, null); when(client.search(request)).thenReturn(response); @@ -415,7 +415,7 @@ PushDownAssertion shouldQuery(QueryBuilder expected) { .size(MAX_RESULT_WINDOW) .timeout(CURSOR_KEEP_ALIVE); OpenSearchRequest request = - new OpenSearchQueryRequest( + OpenSearchQueryRequest.pitOf( EMPLOYEES_INDEX, builder, factory, List.of(), CURSOR_KEEP_ALIVE, null); when(client.search(request)).thenReturn(response); var indexScan = diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java index 81597c65bbe..9258d7bf4db 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java @@ -989,4 +989,85 @@ public void testSortAggregationMetrics2() { + "ORDER BY 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testHaving1() { + String ppl = "source=EMP | stats avg(SAL) as avg by DEPTNO | where avg > 0"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalFilter(condition=[>($0, 0)])\n" + + " LogicalProject(avg=[$1], DEPTNO=[$0])\n" + + " LogicalAggregate(group=[{0}], avg=[AVG($1)])\n" + + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = + "avg=2175.; DEPTNO=20\navg=2916.666666; DEPTNO=10\navg=1566.666666; DEPTNO=30\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT AVG(`SAL`) `avg`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`) `t1`\n" + + "WHERE `avg` > 0"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testHaving2() { + String ppl = + "source=EMP | stats bucket_nullable = false avg(SAL) as avg by DEPTNO | where avg > 0"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalFilter(condition=[>($0, 0)])\n" + + " LogicalProject(avg=[$1], DEPTNO=[$0])\n" + + " LogicalAggregate(group=[{0}], avg=[AVG($1)])\n" + + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = + "avg=2175.; DEPTNO=20\navg=2916.666666; DEPTNO=10\navg=1566.666666; DEPTNO=30\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT AVG(`SAL`) `avg`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `DEPTNO` IS NOT NULL\n" + + "GROUP BY `DEPTNO`) `t2`\n" + + "WHERE `avg` > 0"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testHaving3() { + String ppl = + "source=EMP | stats avg(SAL) as avg, count() as cnt by DEPTNO | eval new_avg = avg + 1000," + + " new_cnt = cnt + 1 | where new_avg > 1000 or new_cnt > 2"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalFilter(condition=[OR(>($3, 1000), >($4, 2))])\n" + + " LogicalProject(avg=[$1], cnt=[$2], DEPTNO=[$0], new_avg=[+($1, 1000)]," + + " new_cnt=[+($2, 1)])\n" + + " LogicalAggregate(group=[{0}], avg=[AVG($1)], cnt=[COUNT()])\n" + + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = + "avg=2175.; cnt=5; DEPTNO=20; new_avg=3175.; new_cnt=6\n" + + "avg=2916.666666; cnt=3; DEPTNO=10; new_avg=3916.666666; new_cnt=4\n" + + "avg=1566.666666; cnt=6; DEPTNO=30; new_avg=2566.666666; new_cnt=7\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT *\n" + + "FROM (SELECT AVG(`SAL`) `avg`, COUNT(*) `cnt`, `DEPTNO`, AVG(`SAL`) + 1000" + + " `new_avg`, COUNT(*) + 1 `new_cnt`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`) `t1`\n" + + "WHERE `new_avg` > 1000 OR `new_cnt` > 2"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } }