From 3dbbcff12a8658a90f9d5e4898d1ceafb328adf8 Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 21 Nov 2023 15:12:58 +0800 Subject: [PATCH 1/3] [opt](nereids) infer result column name in select outfile stmt --- ...t_outfile_expression_generate_col_name.out | 29 ++ ...utfile_expression_generate_col_name.groovy | 260 ++++++++++++++++++ 2 files changed, 289 insertions(+) create mode 100644 regression-test/data/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.out create mode 100644 regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy diff --git a/regression-test/data/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.out b/regression-test/data/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.out new file mode 100644 index 00000000000000..92b36dfae2ee90 --- /dev/null +++ b/regression-test/data/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.out @@ -0,0 +1,29 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_base1 -- +1 id = 1 +10 id not exist +2 id = 2 +3 id not exist +4 id not exist +5 id not exist +6 id not exist +7 id not exist +8 id not exist +9 id not exist + +-- !select_tvf -- +false false +false false +false true +false true +false true +false true +false true +true true +true true +true true + +-- !desc_s3 -- +__case_expr_1 BOOLEAN Yes false \N NONE +id BOOLEAN Yes false \N NONE + diff --git a/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy b/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy new file mode 100644 index 00000000000000..22f94c6c9c0046 --- /dev/null +++ b/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy @@ -0,0 +1,260 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Paths + +suite("test_outfile_expression_generate_col_name", "p0") { + + sql """ set enable_nereids_planner = false """ + sql """ set enable_fallback_to_original_planner = false """ + + String ak = getS3AK() + String sk = getS3SK() + String s3_endpoint = getS3Endpoint() + String region = getS3Region() + String bucket = getS3BucketName(); + + def export_table_name = "outfile_expr_export_test" + def outFilePath = "${bucket}/outfile/expression_generate_col_name/exp_" + + def create_table = { + sql """ DROP TABLE IF EXISTS ${export_table_name} """ + sql """ + CREATE TABLE IF NOT EXISTS outfile_expr_export_test ( + `id` int(11) NULL, + `name` string NULL, + `age` int(11) NULL + ) + PARTITION BY RANGE(id) + ( + PARTITION less_than_20 VALUES LESS THAN ("20"), + PARTITION between_20_70 VALUES [("20"),("70")), + PARTITION more_than_70 VALUES LESS THAN ("151") + ) + DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES("replication_num" = "1"); + """ + } + + def outfile_to_S3 = { format -> + // select ... into outfile ... + def res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${outfile_format} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + + return res[0][3] + } + + // create table to export data + create_table() + + // insert data + StringBuilder sb = new StringBuilder() + int i = 1 + for (; i < 10; i ++) { + sb.append(""" + (${i}, 'doris-${i}', ${i + 18}), + """) + } + sb.append(""" + (${i}, NULL, NULL) + """) + sql """ INSERT INTO ${export_table_name} VALUES + ${sb.toString()} + """ + def insert_res = sql "show last insert;" + logger.info("insert result: " + insert_res.toString()) + order_qt_select_base """ SELECT * FROM ${export_table_name} t ORDER BY id; """ + + + + def check_outfile_data = { outfile_url, outfile_format -> + order_qt_select_tvf """ SELECT * FROM S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } + + def check_outfile_column_name = { outfile_url, outfile_format -> + order_qt_desc_s3 """ Desc function S3 ( + "uri" = "http://${s3_endpoint}${outfile_url.substring(4)}0.${outfile_format}", + "ACCESS_KEY"= "${ak}", + "SECRET_KEY" = "${sk}", + "format" = "${outfile_format}", + "region" = "${region}" + ); + """ + } + + def test_q1 = { outfile_format -> + order_qt_select_base1 """ select 1>2 from ${export_table_name} """ + + // select ... into outfile ... + def res = sql """ + SELECT 1>2 FROM ${export_table_name} + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${outfile_format} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + def outfile_url = res[0][3] + + check_outfile_data(outfile_url, outfile_format) + check_outfile_column_name(outfile_url, outfile_format) + } + + def test_q2 = { outfile_format -> + order_qt_select_base1 """ select max(id) from ${export_table_name} """ + + // select ... into outfile ... + def res = sql """ + SELECT max(id) FROM ${export_table_name} + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${outfile_format} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + def outfile_url = res[0][3] + + check_outfile_data(outfile_url, outfile_format) + check_outfile_column_name(outfile_url, outfile_format) + } + + def test_q3 = { outfile_format -> + order_qt_select_base1 """ select id, case id when 1 then 'id = 1' when 2 then 'id = 2' else 'id not exist' end from ${export_table_name} """ + + // select ... into outfile ... + def res = sql """ + SELECT id, case id when 1 then 'id = 1' when 2 then 'id = 2' else 'id not exist' end FROM ${export_table_name} + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${outfile_format} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + def outfile_url = res[0][3] + println ("outfile_url outfile_url " + outfile_url) + + check_outfile_data(outfile_url, outfile_format) + check_outfile_column_name(outfile_url, outfile_format) + } + + def test_q4 = { outfile_format -> + order_qt_select_base1 """ select + id, + 1, + 'string', + cast (age AS BIGINT), + 1 > 2, + 2 + 3, + 1 IN (1, 2, 3, 4), + TRUE | FALSE + from ${export_table_name} + """ + + // select ... into outfile ... + def res = sql """ + select + id, + 1, + 'string', + cast (age AS BIGINT), + 1 > 2, + 2 + 3, + 1 IN (1, 2, 3, 4), + TRUE | FALSE + from ${export_table_name} + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${outfile_format} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + def outfile_url = res[0][3] + + check_outfile_data(outfile_url, outfile_format) + check_outfile_column_name(outfile_url, outfile_format) + } + + def test_q5 = { outfile_format -> + order_qt_select_base1 """ select cast('2566' as string), cast('888' as bigint), cast('9999' as largeint) + from ${export_table_name} + """ + + // select ... into outfile ... + def res = sql """ + select cast('2566' as string), cast('888' as bigint), cast('9999' as largeint) + from ${export_table_name} + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS ${outfile_format} + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + def outfile_url = res[0][3] + + check_outfile_data(outfile_url, outfile_format) + check_outfile_column_name(outfile_url, outfile_format) + } + + // test parquet format + test_q1("parquet") + test_q2("parquet") + test_q3("parquet") + test_q4("parquet") + test_q5("parquet") + + // test orc format + test_q1("orc") + test_q2("orc") + test_q3("orc") + test_q4("orc") + test_q5("orc") +} From 6ee45c4ded3278045937d9a3beddc6ba788658b0 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 22 Nov 2023 08:00:58 +0800 Subject: [PATCH 2/3] [opt](nereids) infer result column name in select outfile stmt and disable infer name when query --- .../nereids/analyzer/UnboundFileSink.java | 117 ++++++++ .../nereids/glue/LogicalPlanAdapter.java | 3 +- .../nereids/parser/LogicalPlanBuilder.java | 4 +- .../processor/pre/TurnOffPipelineForDml.java | 6 +- .../apache/doris/nereids/rules/RuleType.java | 1 + .../rules/analysis/BindExpression.java | 40 ++- .../trees/plans/visitor/SinkVisitor.java | 5 + ...t_outfile_expression_generate_col_name.out | 262 +++++++++++++++++- ...utfile_expression_generate_col_name.groovy | 2 +- 9 files changed, 411 insertions(+), 29 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFileSink.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFileSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFileSink.java new file mode 100644 index 00000000000000..32a828e94423e8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundFileSink.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.trees.plans.logical.LogicalSink; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * unbound file sink + */ +public class UnboundFileSink extends LogicalSink implements Unbound, Sink { + + private final String filePath; + private final String format; + private final Map properties; + + public UnboundFileSink(CHILD_TYPE child, String filePath, String format, Map properties) { + super(PlanType.LOGICAL_UNBOUND_RESULT_SINK, ImmutableList.of(), child); + this.filePath = filePath; + this.format = format; + this.properties = properties; + } + + public UnboundFileSink(Optional groupExpression, + Optional logicalProperties, CHILD_TYPE child, + String filePath, String format, Map properties) { + super(PlanType.LOGICAL_UNBOUND_RESULT_SINK, ImmutableList.of(), groupExpression, logicalProperties, child); + this.filePath = filePath; + this.format = format; + this.properties = properties; + } + + @Override + public Plan withChildren(List children) { + Preconditions.checkArgument(children.size() == 1, "UnboundFileSink only accepts one child"); + return new UnboundFileSink<>(groupExpression, Optional.empty(), children.get(0), + filePath, format, properties); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitUnboundFileSink(this, context); + } + + @Override + public List getExpressions() { + throw new UnsupportedOperationException(this.getClass().getSimpleName() + " don't support getExpression()"); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return new UnboundFileSink<>(groupExpression, Optional.of(getLogicalProperties()), child(), + filePath, format, properties); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + Preconditions.checkArgument(children.size() == 1, "UnboundFileSink only accepts one child"); + return new UnboundFileSink<>(groupExpression, logicalProperties, children.get(0), + filePath, format, properties); + + } + + @Override + public List computeOutput() { + throw new UnboundException("output"); + } + + @Override + public String toString() { + return Utils.toSqlString("UnboundFileSink[" + id.asInt() + "]"); + } + + public String getFilePath() { + return filePath; + } + + public String getFormat() { + return format; + } + + public Map getProperties() { + return properties; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java index 4d9b70c455efa9..78115a1183b74c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.RedirectStatus; import org.apache.doris.analysis.StatementBase; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.UnboundFileSink; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; @@ -61,7 +62,7 @@ public LogicalPlan getLogicalPlan() { @Override public boolean hasOutFileClause() { - return logicalPlan instanceof LogicalFileSink; + return logicalPlan instanceof LogicalFileSink || logicalPlan instanceof UnboundFileSink; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index bbec310e3df7d9..da381bba9922ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -178,6 +178,7 @@ import org.apache.doris.nereids.DorisParserBaseVisitor; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundAlias; +import org.apache.doris.nereids.analyzer.UnboundFileSink; import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundOlapTableSink; import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; @@ -348,7 +349,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTE; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; -import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; @@ -2334,7 +2334,7 @@ private LogicalPlan withOutFile(LogicalPlan plan, OutFileClauseContext ctx) { properties = visitPropertyClause(ctx.propertyClause()); } Literal filePath = (Literal) visit(ctx.filePath); - return new LogicalFileSink<>(filePath.getStringValue(), format, properties, ImmutableList.of(), plan); + return new UnboundFileSink(plan, filePath.getStringValue(), format, properties); } private LogicalPlan withQueryOrganization(LogicalPlan inputPlan, QueryOrganizationContext ctx) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPipelineForDml.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPipelineForDml.java index 606bc9d69ad6f6..739b2038ad5802 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPipelineForDml.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPipelineForDml.java @@ -20,10 +20,10 @@ import org.apache.doris.analysis.SetVar; import org.apache.doris.analysis.StringLiteral; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.UnboundFileSink; import org.apache.doris.nereids.analyzer.UnboundOlapTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.VariableMgr; @@ -40,9 +40,9 @@ public Plan visitUnboundOlapTableSink(UnboundOlapTableSink unbou } @Override - public Plan visitLogicalFileSink(LogicalFileSink fileSink, StatementContext context) { + public Plan visitUnboundFileSink(UnboundFileSink unboundFileSink, StatementContext context) { turnOffPipeline(context); - return fileSink; + return unboundFileSink; } private void turnOffPipeline(StatementContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index f8760ada9ec93c..88c665c1f62ea8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -30,6 +30,7 @@ public enum RuleType { // **** make sure BINDING_UNBOUND_LOGICAL_PLAN is the lowest priority in the rewrite rules. **** BINDING_RESULT_SINK(RuleTypeClass.REWRITE), + BINDING_FILE_SINK(RuleTypeClass.REWRITE), BINDING_NON_LEAF_LOGICAL_PLAN(RuleTypeClass.REWRITE), BINDING_ONE_ROW_RELATION_SLOT(RuleTypeClass.REWRITE), BINDING_RELATION(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 2608a554fc94d0..ca5b119cf4b935 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; +import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTVFRelation; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -60,6 +61,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; @@ -70,6 +72,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSink; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; @@ -581,22 +584,39 @@ protected boolean condition(Rule rule, Plan plan) { }) ), RuleType.BINDING_RESULT_SINK.build( - unboundResultSink().then(sink -> { - - final ImmutableListMultimap.Builder exprIdToIndexMapBuilder = - ImmutableListMultimap.builder(); - List childOutput = sink.child().getOutput(); - for (int index = 0; index < childOutput.size(); index++) { - exprIdToIndexMapBuilder.put(childOutput.get(index).getExprId(), index); + unboundResultSink().thenApply(ctx -> { + UnboundResultSink unboundResultSink = ctx.root; + if (ctx.connectContext.getState().isQuery()) { + // Should not infer column expression name if query + List outputExprs = unboundResultSink.child().getOutput().stream() + .map(NamedExpression.class::cast) + .collect(ImmutableList.toImmutableList()); + return new LogicalResultSink<>(outputExprs, unboundResultSink.child()); } - InferPlanOutputAlias aliasInfer = new InferPlanOutputAlias(childOutput); - sink.child().accept(aliasInfer, exprIdToIndexMapBuilder.build()); - return new LogicalResultSink<>(aliasInfer.getOutputs(), sink.child()); + return new LogicalResultSink<>(inferColumnNames(unboundResultSink), unboundResultSink.child()); + }) + ), + RuleType.BINDING_FILE_SINK.build( + unboundFileSink().then(sink -> { + return new LogicalFileSink<>(sink.getFilePath(), sink.getFormat(), sink.getProperties(), + inferColumnNames(sink), sink.child()); }) ) ).stream().map(ruleCondition).collect(ImmutableList.toImmutableList()); } + private List inferColumnNames(LogicalSink sink) { + final ImmutableListMultimap.Builder exprIdToIndexMapBuilder = + ImmutableListMultimap.builder(); + List sinkOutput = sink.child().getOutput(); + for (int index = 0; index < sinkOutput.size(); index++) { + exprIdToIndexMapBuilder.put(sinkOutput.get(index).getExprId(), index); + } + InferPlanOutputAlias aliasInfer = new InferPlanOutputAlias(sinkOutput); + sink.child().accept(aliasInfer, exprIdToIndexMapBuilder.build()); + return aliasInfer.getOutputs(); + } + private Plan bindSort(LogicalSort sort, Plan plan, CascadesContext ctx) { // 1. We should deduplicate the slots, otherwise the binding process will fail due to the // ambiguous slots exist. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java index df790fddd29287..36e8801d5cd104 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans.visitor; +import org.apache.doris.nereids.analyzer.UnboundFileSink; import org.apache.doris.nereids.analyzer.UnboundOlapTableSink; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.trees.plans.Plan; @@ -56,6 +57,10 @@ default R visitUnboundResultSink(UnboundResultSink unboundResult return visitLogicalSink(unboundResultSink, context); } + default R visitUnboundFileSink(UnboundFileSink unboundFileSink, C context) { + return visitLogicalSink(unboundFileSink, context); + } + // ******************************* // logical // ******************************* diff --git a/regression-test/data/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.out b/regression-test/data/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.out index 92b36dfae2ee90..f68ee02184097d 100644 --- a/regression-test/data/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.out +++ b/regression-test/data/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.out @@ -1,4 +1,52 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !select_base -- +1 doris-1 19 +10 \N \N +2 doris-2 20 +3 doris-3 21 +4 doris-4 22 +5 doris-5 23 +6 doris-6 24 +7 doris-7 25 +8 doris-8 26 +9 doris-9 27 + +-- !select_base1 -- +false +false +false +false +false +false +false +false +false +false + +-- !select_tvf -- +false +false +false +false +false +false +false +false +false +false + +-- !desc_s3 -- +__greater_than_0 BOOLEAN Yes false \N NONE + +-- !select_base1 -- +10 + +-- !select_tvf -- +10 + +-- !desc_s3 -- +__max_0 INT Yes false \N NONE + -- !select_base1 -- 1 id = 1 10 id not exist @@ -12,18 +60,208 @@ 9 id not exist -- !select_tvf -- -false false -false false -false true -false true -false true -false true -false true -true true -true true -true true +1 id = 1 +10 id not exist +2 id = 2 +3 id not exist +4 id not exist +5 id not exist +6 id not exist +7 id not exist +8 id not exist +9 id not exist + +-- !desc_s3 -- +__case_when_1 TEXT Yes false \N NONE +id INT Yes false \N NONE + +-- !select_base1 -- +1 1 string 19 false 5 true 1 +10 1 string \N false 5 true 1 +2 1 string 20 false 5 true 1 +3 1 string 21 false 5 true 1 +4 1 string 22 false 5 true 1 +5 1 string 23 false 5 true 1 +6 1 string 24 false 5 true 1 +7 1 string 25 false 5 true 1 +8 1 string 26 false 5 true 1 +9 1 string 27 false 5 true 1 + +-- !select_tvf -- +1 1 string 19 false 5 true 1 +10 1 string \N false 5 true 1 +2 1 string 20 false 5 true 1 +3 1 string 21 false 5 true 1 +4 1 string 22 false 5 true 1 +5 1 string 23 false 5 true 1 +6 1 string 24 false 5 true 1 +7 1 string 25 false 5 true 1 +8 1 string 26 false 5 true 1 +9 1 string 27 false 5 true 1 + +-- !desc_s3 -- +__add_5 INT Yes false \N NONE +__bit_or_7 INT Yes false \N NONE +__cast_3 BOOLEAN Yes false \N NONE\ +__greater_than_4 BIGINT Yes false \N NONE +__in_predicate_6 BOOLEAN Yes false \N NONE +__literal_1 INT Yes false \N NONE +__literal_2 TEXT Yes false \N NONE +id INT Yes false \N NONE + +-- !select_base1 -- +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 + +-- !select_tvf -- +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 + +-- !desc_s3 -- +__cast_0 TEXT Yes false \N NONE +__cast_1 BIGINT Yes false \N NONE +__cast_2 TEXT Yes false \N NONE + +-- !select_base1 -- +false +false +false +false +false +false +false +false +false +false + +-- !select_tvf -- +false +false +false +false +false +false +false +false +false +false + +-- !desc_s3 -- +__binary_predicate_0 BOOLEAN Yes false \N NONE + +-- !select_base1 -- +10 + +-- !select_tvf -- +10 + +-- !desc_s3 -- +__max_0 INT Yes false \N NONE + +-- !select_base1 -- +1 id = 1 +10 id not exist +2 id = 2 +3 id not exist +4 id not exist +5 id not exist +6 id not exist +7 id not exist +8 id not exist +9 id not exist + +-- !select_tvf -- +1 id = 1 +10 id not exist +2 id = 2 +3 id not exist +4 id not exist +5 id not exist +6 id not exist +7 id not exist +8 id not exist +9 id not exist + +-- !desc_s3 -- +__case_expr_1 TEXT Yes false \N NONE +id INT Yes false \N NONE + +-- !select_base1 -- +1 1 string 19 false 5 true 1 +10 1 string \N false 5 true 1 +2 1 string 20 false 5 true 1 +3 1 string 21 false 5 true 1 +4 1 string 22 false 5 true 1 +5 1 string 23 false 5 true 1 +6 1 string 24 false 5 true 1 +7 1 string 25 false 5 true 1 +8 1 string 26 false 5 true 1 +9 1 string 27 false 5 true 1 + +-- !select_tvf -- +1 1 string 19 false 5 true 1 +10 1 string \N false 5 true 1 +2 1 string 20 false 5 true 1 +3 1 string 21 false 5 true 1 +4 1 string 22 false 5 true 1 +5 1 string 23 false 5 true 1 +6 1 string 24 false 5 true 1 +7 1 string 25 false 5 true 1 +8 1 string 26 false 5 true 1 +9 1 string 27 false 5 true 1 + +-- !desc_s3 -- +__arithmetic_expr_5 SMALLINT Yes false \N NONE +__arithmetic_expr_7 BIGINT Yes false \N NONE +__binary_predicate_4 BOOLEAN Yes false \N NONE +__cast_expr_3 BIGINT Yes false \N NONE +__in_predicate_6 BOOLEAN Yes false \N NONE +__literal_1 TINYINT Yes false \N NONE +__literal_2 TEXT Yes false \N NONE +id INT Yes false \N NONE + +-- !select_base1 -- +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 + +-- !select_tvf -- +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 +2566 888 9999 -- !desc_s3 -- -__case_expr_1 BOOLEAN Yes false \N NONE -id BOOLEAN Yes false \N NONE +__cast_expr_0 TEXT Yes false \N NONE +__cast_expr_1 BIGINT Yes false \N NONE +__cast_expr_2 TEXT Yes false \N NONE diff --git a/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy b/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy index 22f94c6c9c0046..838ee144136bb2 100644 --- a/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy +++ b/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy @@ -23,7 +23,7 @@ import java.nio.file.Paths suite("test_outfile_expression_generate_col_name", "p0") { - sql """ set enable_nereids_planner = false """ + sql """ set enable_nereids_planner = true """ sql """ set enable_fallback_to_original_planner = false """ String ak = getS3AK() From 14cea6a4b11ad1e3605659a827e120d24f376390 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 22 Nov 2023 23:26:49 +0800 Subject: [PATCH 3/3] modify out file path --- .../test_outfile_expression_generate_col_name.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy b/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy index 838ee144136bb2..096b26f51a102c 100644 --- a/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy +++ b/regression-test/suites/export_p0/outfile/outfile_expr/test_outfile_expression_generate_col_name.groovy @@ -33,7 +33,7 @@ suite("test_outfile_expression_generate_col_name", "p0") { String bucket = getS3BucketName(); def export_table_name = "outfile_expr_export_test" - def outFilePath = "${bucket}/outfile/expression_generate_col_name/exp_" + def outFilePath = "${bucket}/outfile/expr_generate_col_name/exp_" def create_table = { sql """ DROP TABLE IF EXISTS ${export_table_name} """