From 330d9b006418bf8c48600c088de1642dd9593207 Mon Sep 17 00:00:00 2001 From: sohardforaname Date: Mon, 7 Aug 2023 14:42:48 +0800 Subject: [PATCH 01/11] 'fix-bugs' --- .../main/java/org/apache/doris/planner/SingleNodePlanner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index e3fa6978b54808..faeb6183c009be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -1055,7 +1055,7 @@ private PlanNode createJoinPlan(Analyzer analyzer, TableRef leftmostRef, List Date: Wed, 9 Aug 2023 14:36:03 +0800 Subject: [PATCH 02/11] 'fix-bugs' --- .../java/org/apache/doris/planner/OriginalPlanner.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 08af81afbb19cd..6e0bbeeb2a34a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -264,6 +264,15 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue Collections.reverse(fragments); + // the nullable mode will be changed in create plan fragment, we should adjust the ones in querystmt's result + // exprs. + PlanFragment root = fragments.get(0); + List slots = root.getPlanRoot().getOutputTupleDesc().getSlots(); + Preconditions.checkArgument(queryStmt.getResultExprs().size() == slots.size()); + for (int i = 0; i < slots.size(); ++i) { + queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn().setIsAllowNull(slots.get(i).getIsNullable()); + } + pushDownResultFileSink(analyzer); pushOutColumnUniqueIdsToOlapScan(rootFragment, analyzer); From 24c67f4769aa73813980e4e19f612b013a2e60ec Mon Sep 17 00:00:00 2001 From: sohardforaname Date: Wed, 9 Aug 2023 14:36:28 +0800 Subject: [PATCH 03/11] 'fix-bugs' --- regression-test/suites/delete_p0/test_delete.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/delete_p0/test_delete.groovy b/regression-test/suites/delete_p0/test_delete.groovy index c19884b411fe9e..bcd648c2fdc204 100644 --- a/regression-test/suites/delete_p0/test_delete.groovy +++ b/regression-test/suites/delete_p0/test_delete.groovy @@ -233,7 +233,7 @@ suite("test_delete") { PROPERTIES ( "replication_num" = "1", "enable_unique_key_merge_on_write" = "true" - ); + ); ''' sql 'insert into test1 values("a", "a"), ("bb", "bb"), ("ccc", "ccc")' From ad8d4b90cae01a3f22a5cdb3c6f7b51c5ea365a5 Mon Sep 17 00:00:00 2001 From: sohardforaname Date: Wed, 9 Aug 2023 14:51:07 +0800 Subject: [PATCH 04/11] 'fix-bugs' --- .../org/apache/doris/planner/OriginalPlanner.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 6e0bbeeb2a34a6..4adaa2ebdd50a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -266,11 +266,14 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue // the nullable mode will be changed in create plan fragment, we should adjust the ones in querystmt's result // exprs. - PlanFragment root = fragments.get(0); - List slots = root.getPlanRoot().getOutputTupleDesc().getSlots(); - Preconditions.checkArgument(queryStmt.getResultExprs().size() == slots.size()); - for (int i = 0; i < slots.size(); ++i) { - queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn().setIsAllowNull(slots.get(i).getIsNullable()); + if (queryStmt instanceof SelectStmt) { + PlanFragment root = fragments.get(0); + List slots = root.getOutputExprs(); + Preconditions.checkArgument(queryStmt.getResultExprs().size() == slots.size()); + for (int i = 0; i < slots.size(); ++i) { + queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn() + .setIsAllowNull(slots.get(i).isNullable()); + } } pushDownResultFileSink(analyzer); From 91039ab33ecce8f81f5744c2e955a0c5d1738dd8 Mon Sep 17 00:00:00 2001 From: sohardforaname Date: Wed, 9 Aug 2023 14:59:36 +0800 Subject: [PATCH 05/11] 'fix-bugs' --- .../main/java/org/apache/doris/planner/OriginalPlanner.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 4adaa2ebdd50a9..8324be106cff1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -271,8 +271,10 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue List slots = root.getOutputExprs(); Preconditions.checkArgument(queryStmt.getResultExprs().size() == slots.size()); for (int i = 0; i < slots.size(); ++i) { - queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn() - .setIsAllowNull(slots.get(i).isNullable()); + if (queryStmt.getResultExprs().get(i).getSrcSlotRef() != null) { + queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn() + .setIsAllowNull(slots.get(i).isNullable()); + } } } From 102fedbb82dc5351b7fd9eea658b35355ccfc34c Mon Sep 17 00:00:00 2001 From: sohardforaname Date: Wed, 9 Aug 2023 15:13:45 +0800 Subject: [PATCH 06/11] 'fix-bugs' --- .../main/java/org/apache/doris/planner/OriginalPlanner.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 8324be106cff1a..55ecee370dc4fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -268,12 +268,14 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue // exprs. if (queryStmt instanceof SelectStmt) { PlanFragment root = fragments.get(0); - List slots = root.getOutputExprs(); + List slots = analyzer.getDescTbl() + .getTupleDesc(root.getPlanRoot().getOutputTupleIds().get(0)) + .getSlots(); Preconditions.checkArgument(queryStmt.getResultExprs().size() == slots.size()); for (int i = 0; i < slots.size(); ++i) { if (queryStmt.getResultExprs().get(i).getSrcSlotRef() != null) { queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn() - .setIsAllowNull(slots.get(i).isNullable()); + .setIsAllowNull(slots.get(i).getIsNullable()); } } } From 03b4b8cc0d5c792aa080cd3ad4b6a38f01b3f881 Mon Sep 17 00:00:00 2001 From: sohardforaname Date: Wed, 9 Aug 2023 15:31:07 +0800 Subject: [PATCH 07/11] 'fix-bugs' --- .../suites/ddl_p0/test_ctas.groovy | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/regression-test/suites/ddl_p0/test_ctas.groovy b/regression-test/suites/ddl_p0/test_ctas.groovy index 35bd4f2726a52e..7c093aa4793593 100644 --- a/regression-test/suites/ddl_p0/test_ctas.groovy +++ b/regression-test/suites/ddl_p0/test_ctas.groovy @@ -205,6 +205,39 @@ suite("test_ctas") { DROP TABLE IF EXISTS ctas_113815 """ } + + try { + sql '''create table a ( + id int not null, + name varchar(20) not null + ) + distributed by hash(id) buckets 4 + properties ( + "replication_num"="1" + ); + ''' + + sql '''create table b ( + id int not null, + age int not null + ) + distributed by hash(id) buckets 4 + properties ( + "replication_num"="1" + ); + ''' + + sql 'insert into a values(1, \'ww\'), (2, \'zs\');' + sql 'insert into b values(1, 22);' + sql 'create table c properties("replication_num"="1") as select a.id, a.name, b.age from a left join b on a.id = b.id;' + + String desc = sql 'desc c' + assertTrue(desc.contains('Yes')) + } finally { + sql 'drop table a' + sql 'drop table b' + sql 'drop table c' + } } From 3207b9b5075e13cfa443198a50495d5c4311d2e5 Mon Sep 17 00:00:00 2001 From: sohardforaname Date: Mon, 14 Aug 2023 12:23:56 +0800 Subject: [PATCH 08/11] 'fix-bugs' --- .../org/apache/doris/planner/OriginalPlanner.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 55ecee370dc4fb..bbb36b330c0680 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -271,11 +271,13 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue List slots = analyzer.getDescTbl() .getTupleDesc(root.getPlanRoot().getOutputTupleIds().get(0)) .getSlots(); - Preconditions.checkArgument(queryStmt.getResultExprs().size() == slots.size()); - for (int i = 0; i < slots.size(); ++i) { - if (queryStmt.getResultExprs().get(i).getSrcSlotRef() != null) { - queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn() - .setIsAllowNull(slots.get(i).getIsNullable()); + // to exclude the cases that outputs contain an expression. + if (queryStmt.getResultExprs().size() == slots.size()) { + for (int i = 0; i < slots.size(); ++i) { + if (queryStmt.getResultExprs().get(i).getSrcSlotRef() != null) { + queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn() + .setIsAllowNull(slots.get(i).getIsNullable()); + } } } } From 6ab62d285973db2196a67dd781dd5782cb01a810 Mon Sep 17 00:00:00 2001 From: sohardforaname Date: Tue, 15 Aug 2023 13:30:25 +0800 Subject: [PATCH 09/11] 'fix-bugs' --- .../doris/analysis/CreateTableAsSelectStmt.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java index 2d6ee678db3a52..f3eff6677dc7d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java @@ -20,7 +20,11 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; +import org.apache.doris.planner.OriginalPlanner; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.qe.ConnectContext; +import com.google.common.base.Preconditions; import lombok.Getter; import java.util.ArrayList; @@ -67,6 +71,18 @@ public void analyze(Analyzer analyzer) throws UserException { QueryStmt tmpStmt = queryStmt.clone(); tmpStmt.analyze(dummyRootAnalyzer); this.queryStmt = tmpStmt; + // to adjust the nullable of the result expression, we have to create plan fragment from the query stmt. + OriginalPlanner planner = new OriginalPlanner(dummyRootAnalyzer); + planner.createPlanFragments(queryStmt, dummyRootAnalyzer, ConnectContext.get().getSessionVariable().toThrift()); + PlanFragment root = planner.getFragments().get(0); + List outputs = root.getOutputExprs(); + Preconditions.checkArgument(outputs.size() == queryStmt.getResultExprs().size()); + for (int i = 0; i < outputs.size(); ++i) { + if (queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn() != null) { + queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn() + .setIsAllowNull(outputs.get(i).isNullable()); + } + } ArrayList resultExprs = getQueryStmt().getResultExprs(); if (columnNames != null && columnNames.size() != resultExprs.size()) { ErrorReport.reportAnalysisException(ErrorCode.ERR_COL_NUMBER_NOT_MATCH); From 0b8d8e9b834969001b6b90abfeea34c77a0318b3 Mon Sep 17 00:00:00 2001 From: sohardforaname Date: Tue, 15 Aug 2023 19:21:54 +0800 Subject: [PATCH 10/11] 'fix-bugs' --- .../java/org/apache/doris/analysis/CreateTableAsSelectStmt.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java index f3eff6677dc7d8..942c3cce70d6df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java @@ -78,7 +78,7 @@ public void analyze(Analyzer analyzer) throws UserException { List outputs = root.getOutputExprs(); Preconditions.checkArgument(outputs.size() == queryStmt.getResultExprs().size()); for (int i = 0; i < outputs.size(); ++i) { - if (queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn() != null) { + if (queryStmt.getResultExprs().get(i).getSrcSlotRef() != null) { queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn() .setIsAllowNull(outputs.get(i).isNullable()); } From a3963241959ad4704adb6a9f23a6c35a8b839ffe Mon Sep 17 00:00:00 2001 From: sohardforaname Date: Wed, 16 Aug 2023 00:16:30 +0800 Subject: [PATCH 11/11] 'fix-bugs' --- .../apache/doris/planner/OriginalPlanner.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index bbb36b330c0680..08af81afbb19cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -264,24 +264,6 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue Collections.reverse(fragments); - // the nullable mode will be changed in create plan fragment, we should adjust the ones in querystmt's result - // exprs. - if (queryStmt instanceof SelectStmt) { - PlanFragment root = fragments.get(0); - List slots = analyzer.getDescTbl() - .getTupleDesc(root.getPlanRoot().getOutputTupleIds().get(0)) - .getSlots(); - // to exclude the cases that outputs contain an expression. - if (queryStmt.getResultExprs().size() == slots.size()) { - for (int i = 0; i < slots.size(); ++i) { - if (queryStmt.getResultExprs().get(i).getSrcSlotRef() != null) { - queryStmt.getResultExprs().get(i).getSrcSlotRef().getColumn() - .setIsAllowNull(slots.get(i).getIsNullable()); - } - } - } - } - pushDownResultFileSink(analyzer); pushOutColumnUniqueIdsToOlapScan(rootFragment, analyzer);