diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java index 10c82a906c4930..7ded9ba56c1a7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -655,7 +655,7 @@ private TupleDescriptor resolveColumnRef(TableName tblName, String colName) thro // find table all name for (TupleDescriptor desc : tupleByAlias.get(tblName.toString())) { //result = desc; - if (!isVisible(desc.getId())) { + if (!colName.equalsIgnoreCase(Column.DELETE_SIGN) && !isVisible(desc.getId())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_ILLEGAL_COLUMN_REFERENCE_ERROR, Joiner.on(".").join(tblName.getTbl(),colName)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index f640c8d456f89d..b9abe316d97141 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -22,7 +22,6 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.FunctionSet; -import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table.TableType; import org.apache.doris.catalog.Type; @@ -340,7 +339,6 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { return; } super.analyze(analyzer); - fromClause_.setNeedToSql(needToSql); fromClause_.analyze(analyzer); @@ -533,21 +531,6 @@ private void whereClauseRewrite() { whereClause = new BoolLiteral(true); } } - // filter deleted data by and DELETE_SIGN column = 0, DELETE_SIGN is a hidden column - // that indicates whether the row is delete - for (TableRef tableRef : fromClause_.getTableRefs()) { - if (!isForbiddenMVRewrite() && tableRef instanceof BaseTableRef && tableRef.getTable() instanceof OlapTable - && ((OlapTable) tableRef.getTable()).getKeysType() == KeysType.UNIQUE_KEYS - && ((OlapTable) tableRef.getTable()).hasDeleteSign()) { - BinaryPredicate filterDeleteExpr = new BinaryPredicate(BinaryPredicate.Operator.EQ, - new SlotRef(tableRef.getName(), Column.DELETE_SIGN), new IntLiteral(0)); - if (whereClause == null) { - whereClause = filterDeleteExpr; - } else { - whereClause = new CompoundPredicate(CompoundPredicate.Operator.AND, filterDeleteExpr, whereClause); - } - } - } Expr deDuplicatedWhere = deduplicateOrs(whereClause); if (deDuplicatedWhere != null) { whereClause = deDuplicatedWhere; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 24ee4a33a0a154..81964fdbf47cc8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -33,6 +33,7 @@ import org.apache.doris.analysis.GroupingInfo; import org.apache.doris.analysis.InPredicate; import org.apache.doris.analysis.InlineViewRef; +import org.apache.doris.analysis.IntLiteral; import org.apache.doris.analysis.IsNullPredicate; import org.apache.doris.analysis.JoinOperator; import org.apache.doris.analysis.LiteralExpr; @@ -52,6 +53,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.MysqlTable; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; @@ -1361,6 +1363,13 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s switch (tblRef.getTable().getType()) { case OLAP: OlapScanNode olapNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), "OlapScanNode"); + if (((OlapTable) tblRef.getTable()).hasDeleteSign()) { + Expr conjunct = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new SlotRef(tblRef.getAliasAsName(), Column.DELETE_SIGN), new IntLiteral(0)); + conjunct.analyze(analyzer); + analyzer.registerConjunct(conjunct, tblRef.getDesc().getId()); + } + olapNode.setForceOpenPreAgg(tblRef.isForcePreAggOpened()); scanNode = olapNode; break; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java index 278e69d64eceeb..74e736a8ec88bf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java @@ -64,11 +64,42 @@ public static void setUp() throws Exception { "\"storage_type\" = \"COLUMN\",\n" + "\"replication_num\" = \"1\"\n" + ");"; + + String tbl1 = "CREATE TABLE db1.table1 (\n" + + " `siteid` int(11) NULL DEFAULT \"10\" COMMENT \"\",\n" + + " `citycode` smallint(6) NULL COMMENT \"\",\n" + + " `username` varchar(32) NULL DEFAULT \"\" COMMENT \"\",\n" + + " `pv` bigint(20) NULL DEFAULT \"0\" COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "UNIQUE KEY(`siteid`, `citycode`, `username`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`siteid`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ")"; + String tbl2 = "CREATE TABLE db1.table2 (\n" + + " `siteid` int(11) NULL DEFAULT \"10\" COMMENT \"\",\n" + + " `citycode` smallint(6) NULL COMMENT \"\",\n" + + " `username` varchar(32) NULL DEFAULT \"\" COMMENT \"\",\n" + + " `pv` bigint(20) NULL DEFAULT \"0\" COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "UNIQUE KEY(`siteid`, `citycode`, `username`)\n" + + "COMMENT \"OLAP\"\n" + + "DISTRIBUTED BY HASH(`siteid`) BUCKETS 10\n" + + "PROPERTIES (\n" + + "\"replication_num\" = \"1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\"\n" + + ")"; dorisAssert = new DorisAssert(); dorisAssert.withDatabase("db1").useDatabase("db1"); dorisAssert.withTable(createTblStmtStr) .withTable(createBaseAllStmtStr) - .withTable(createPratitionTableStr); + .withTable(createPratitionTableStr) + .withTable(tbl1) + .withTable(tbl2); } @Test @@ -396,4 +427,15 @@ public void testVarcharToLongSupport() throws Exception { .explainQuery() .contains("`datekey` = 20200730")); } + @Test + public void testDeleteSign() throws Exception { + String sql1 = "SELECT * FROM db1.table1 LEFT ANTI JOIN db1.table2 ON db1.table1.siteid = db1.table2.siteid;"; + Assert.assertTrue(dorisAssert.query(sql1).explainQuery().contains("`table1`.`__DORIS_DELETE_SIGN__` = 0")); + String sql2 = "SELECT * FROM db1.table1 JOIN db1.table2 ON db1.table1.siteid = db1.table2.siteid;"; + Assert.assertTrue(dorisAssert.query(sql2).explainQuery().contains("`table1`.`__DORIS_DELETE_SIGN__` = 0")); + String sql3 = "SELECT * FROM table1"; + Assert.assertTrue(dorisAssert.query(sql3).explainQuery().contains("`table1`.`__DORIS_DELETE_SIGN__` = 0")); + String sql4 = " SELECT * FROM table1 table2"; + Assert.assertTrue(dorisAssert.query(sql4).explainQuery().contains("`table2`.`__DORIS_DELETE_SIGN__` = 0")); + } }