Skip to content

fix semi/anti join error when table has delete sign column #4498

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 6 commits into from
Sep 3, 2020
Merged

fix semi/anti join error when table has delete sign column #4498

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a9596d6
fix semi/anti join error when table has delete sign column
yangzhg Sep 1, 2020
0f0cc63
move filter to scanNode
yangzhg Sep 1, 2020
1a2df8f
move filter to scanNode
yangzhg Sep 1, 2020
6e9cd93
move filter to scanNode
yangzhg Sep 1, 2020
05c6f79
remove unused exceptions
yangzhg Sep 2, 2020
7405891
fix bug
yangzhg Sep 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -655,7 +655,7 @@ private TupleDescriptor resolveColumnRef(TableName tblName, String colName) thro
// find table all name
for (TupleDescriptor desc : tupleByAlias.get(tblName.toString())) {
//result = desc;
if (!isVisible(desc.getId())) {
if (!colName.equalsIgnoreCase(Column.DELETE_SIGN) && !isVisible(desc.getId())) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_ILLEGAL_COLUMN_REFERENCE_ERROR,
Joiner.on(".").join(tblName.getTbl(),colName));
}
Expand Down
17 changes: 0 additions & 17 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table.TableType;
import org.apache.doris.catalog.Type;
Expand Down Expand Up @@ -340,7 +339,6 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
return;
}
super.analyze(analyzer);

fromClause_.setNeedToSql(needToSql);
fromClause_.analyze(analyzer);

Expand Down Expand Up @@ -533,21 +531,6 @@ private void whereClauseRewrite() {
whereClause = new BoolLiteral(true);
}
}
// filter deleted data by and DELETE_SIGN column = 0, DELETE_SIGN is a hidden column
// that indicates whether the row is delete
for (TableRef tableRef : fromClause_.getTableRefs()) {
if (!isForbiddenMVRewrite() && tableRef instanceof BaseTableRef && tableRef.getTable() instanceof OlapTable
&& ((OlapTable) tableRef.getTable()).getKeysType() == KeysType.UNIQUE_KEYS
&& ((OlapTable) tableRef.getTable()).hasDeleteSign()) {
BinaryPredicate filterDeleteExpr = new BinaryPredicate(BinaryPredicate.Operator.EQ,
new SlotRef(tableRef.getName(), Column.DELETE_SIGN), new IntLiteral(0));
if (whereClause == null) {
whereClause = filterDeleteExpr;
} else {
whereClause = new CompoundPredicate(CompoundPredicate.Operator.AND, filterDeleteExpr, whereClause);
}
}
}
Expr deDuplicatedWhere = deduplicateOrs(whereClause);
if (deDuplicatedWhere != null) {
whereClause = deDuplicatedWhere;
Expand Down
9 changes: 9 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.doris.analysis.GroupingInfo;
import org.apache.doris.analysis.InPredicate;
import org.apache.doris.analysis.InlineViewRef;
import org.apache.doris.analysis.IntLiteral;
import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.JoinOperator;
import org.apache.doris.analysis.LiteralExpr;
Expand All @@ -52,6 +53,7 @@
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.MysqlTable;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
Expand Down Expand Up @@ -1361,6 +1363,13 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s
switch (tblRef.getTable().getType()) {
case OLAP:
OlapScanNode olapNode = new OlapScanNode(ctx_.getNextNodeId(), tblRef.getDesc(), "OlapScanNode");
if (((OlapTable) tblRef.getTable()).hasDeleteSign()) {
Expr conjunct = new BinaryPredicate(BinaryPredicate.Operator.EQ,
new SlotRef(tblRef.getAliasAsName(), Column.DELETE_SIGN), new IntLiteral(0));
conjunct.analyze(analyzer);
analyzer.registerConjunct(conjunct, tblRef.getDesc().getId());
}

olapNode.setForceOpenPreAgg(tblRef.isForcePreAggOpened());
scanNode = olapNode;
break;
Expand Down
44 changes: 43 additions & 1 deletion fe/fe-core/src/test/java/org/apache/doris/analysis/SelectStmtTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,42 @@ public static void setUp() throws Exception {
"\"storage_type\" = \"COLUMN\",\n" +
"\"replication_num\" = \"1\"\n" +
");";

String tbl1 = "CREATE TABLE db1.table1 (\n" +
" `siteid` int(11) NULL DEFAULT \"10\" COMMENT \"\",\n" +
" `citycode` smallint(6) NULL COMMENT \"\",\n" +
" `username` varchar(32) NULL DEFAULT \"\" COMMENT \"\",\n" +
" `pv` bigint(20) NULL DEFAULT \"0\" COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
"UNIQUE KEY(`siteid`, `citycode`, `username`)\n" +
"COMMENT \"OLAP\"\n" +
"DISTRIBUTED BY HASH(`siteid`) BUCKETS 10\n" +
"PROPERTIES (\n" +
"\"replication_num\" = \"1\",\n" +
"\"in_memory\" = \"false\",\n" +
"\"storage_format\" = \"V2\"\n" +
")";
String tbl2 = "CREATE TABLE db1.table2 (\n" +
" `siteid` int(11) NULL DEFAULT \"10\" COMMENT \"\",\n" +
" `citycode` smallint(6) NULL COMMENT \"\",\n" +
" `username` varchar(32) NULL DEFAULT \"\" COMMENT \"\",\n" +
" `pv` bigint(20) NULL DEFAULT \"0\" COMMENT \"\"\n" +
") ENGINE=OLAP\n" +
"UNIQUE KEY(`siteid`, `citycode`, `username`)\n" +
"COMMENT \"OLAP\"\n" +
"DISTRIBUTED BY HASH(`siteid`) BUCKETS 10\n" +
"PROPERTIES (\n" +
"\"replication_num\" = \"1\",\n" +
"\"in_memory\" = \"false\",\n" +
"\"storage_format\" = \"V2\"\n" +
")";
dorisAssert = new DorisAssert();
dorisAssert.withDatabase("db1").useDatabase("db1");
dorisAssert.withTable(createTblStmtStr)
.withTable(createBaseAllStmtStr)
.withTable(createPratitionTableStr);
.withTable(createPratitionTableStr)
.withTable(tbl1)
.withTable(tbl2);
}

@Test
Expand Down Expand Up @@ -396,4 +427,15 @@ public void testVarcharToLongSupport() throws Exception {
.explainQuery()
.contains("`datekey` = 20200730"));
}
@Test
public void testDeleteSign() throws Exception {
String sql1 = "SELECT * FROM db1.table1 LEFT ANTI JOIN db1.table2 ON db1.table1.siteid = db1.table2.siteid;";
Assert.assertTrue(dorisAssert.query(sql1).explainQuery().contains("`table1`.`__DORIS_DELETE_SIGN__` = 0"));
String sql2 = "SELECT * FROM db1.table1 JOIN db1.table2 ON db1.table1.siteid = db1.table2.siteid;";
Assert.assertTrue(dorisAssert.query(sql2).explainQuery().contains("`table1`.`__DORIS_DELETE_SIGN__` = 0"));
String sql3 = "SELECT * FROM table1";
Assert.assertTrue(dorisAssert.query(sql3).explainQuery().contains("`table1`.`__DORIS_DELETE_SIGN__` = 0"));
String sql4 = " SELECT * FROM table1 table2";
Assert.assertTrue(dorisAssert.query(sql4).explainQuery().contains("`table2`.`__DORIS_DELETE_SIGN__` = 0"));
}
}