From 769e5ea9ef22073cec524bc87c7047d3b3477802 Mon Sep 17 00:00:00 2001 From: JingDas Date: Wed, 11 Oct 2023 18:24:06 +0800 Subject: [PATCH 1/7] [fix](meta) Disable infer expr column name when query on old optimizer --- .../java/org/apache/doris/analysis/Analyzer.java | 12 ++++++++++++ .../java/org/apache/doris/analysis/SelectStmt.java | 10 +++++++++- .../main/java/org/apache/doris/qe/StmtExecutor.java | 2 ++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java index f0e96665096792..9571ef505b88a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -319,6 +319,10 @@ private static class GlobalState { // analysis. public boolean isExplain; + // Record the statement clazz that the analyzer would to analyze, + // this give an opportunity to control analyzing behavior according to the statement type. + public Class statementClazz; + // Indicates whether the query has plan hints. public boolean hasPlanHints = false; @@ -567,6 +571,14 @@ public boolean isExplain() { return globalState.isExplain; } + public void setStatementClazz(Class statementClazz) { + globalState.statementClazz = statementClazz; + } + + public Class getStatementClazz() { + return globalState.statementClazz; + } + public int incrementCallDepth() { return ++callDepth; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index 47478ba3d7f688..04ae73812cd8ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -543,7 +543,15 @@ public void analyze(Analyzer analyzer) throws UserException { throw new AnalysisException("Subquery is not supported in the select list."); } resultExprs.add(rewriteQueryExprByMvColumnExpr(item.getExpr(), analyzer)); - String columnLabel = item.toColumnLabel(i); + String columnLabel = null; + Class statementClazz = analyzer.getStatementClazz(); + if (statementClazz != null && (CreateViewStmt.class.isAssignableFrom(statementClazz) || + CreateMaterializedViewStmt.class.isAssignableFrom(statementClazz))) { + columnLabel = item.toColumnLabel(i); + } + if (columnLabel == null) { + columnLabel = item.toColumnLabel(); + } SlotRef aliasRef = new SlotRef(null, columnLabel); Expr existingAliasExpr = aliasSMap.get(aliasRef); if (existingAliasExpr != null && !existingAliasExpr.equals(item.getExpr())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 2ce0de22cb2d6c..f243fb30e6290d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -719,6 +719,7 @@ public void executeByLegacy(TUniqueId queryId) throws Exception { } } else { analyzer = new Analyzer(context.getEnv(), context); + analyzer.setStatementClazz(parsedStmt.getClass()); parsedStmt.analyze(analyzer); } @@ -933,6 +934,7 @@ public void analyze(TQueryOptions tQueryOptions) throws UserException, Interrupt } analyzer = new Analyzer(context.getEnv(), context); + analyzer.setStatementClazz(parsedStmt.getClass()); if (parsedStmt instanceof PrepareStmt || context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { From 95ec9c9caa67eb66028b5493ced31a5bb0ac793a Mon Sep 17 00:00:00 2001 From: JingDas Date: Thu, 12 Oct 2023 00:33:26 +0800 Subject: [PATCH 2/7] [fix](meta) support to infer column name when create table as --- .../main/java/org/apache/doris/analysis/SelectListItem.java | 1 - .../src/main/java/org/apache/doris/analysis/SelectStmt.java | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectListItem.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectListItem.java index 8fc99e29673d89..34368c7b17eaf7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectListItem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectListItem.java @@ -120,7 +120,6 @@ public String toDigest() { * Return a column label for the select list item. Without generate column name * automatically. */ - @Deprecated public String toColumnLabel() { Preconditions.checkState(!isStar()); if (alias != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index 04ae73812cd8ee..fd76d70332a971 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -546,10 +546,12 @@ public void analyze(Analyzer analyzer) throws UserException { String columnLabel = null; Class statementClazz = analyzer.getStatementClazz(); if (statementClazz != null && (CreateViewStmt.class.isAssignableFrom(statementClazz) || - CreateMaterializedViewStmt.class.isAssignableFrom(statementClazz))) { + CreateMaterializedViewStmt.class.isAssignableFrom(statementClazz) || + CreateTableAsSelectStmt.class.isAssignableFrom(statementClazz))) { columnLabel = item.toColumnLabel(i); } if (columnLabel == null) { + // column label without position is applicative for query columnLabel = item.toColumnLabel(); } SlotRef aliasRef = new SlotRef(null, columnLabel); From cd11464da21caaf47deabd82361cb8b00fd26a76 Mon Sep 17 00:00:00 2001 From: JingDas Date: Thu, 12 Oct 2023 18:18:46 +0800 Subject: [PATCH 3/7] [fix](meta) fix ut test fail --- .../org/apache/doris/analysis/SelectStmt.java | 4 +- .../apache/doris/analysis/StatementBase.java | 3 + .../org/apache/doris/qe/StmtExecutor.java | 2 - .../apache/doris/qe/OlapQueryCacheTest.java | 111 +++++++++++------- 4 files changed, 72 insertions(+), 48 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index fd76d70332a971..eeb0334e6db7f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -545,9 +545,7 @@ public void analyze(Analyzer analyzer) throws UserException { resultExprs.add(rewriteQueryExprByMvColumnExpr(item.getExpr(), analyzer)); String columnLabel = null; Class statementClazz = analyzer.getStatementClazz(); - if (statementClazz != null && (CreateViewStmt.class.isAssignableFrom(statementClazz) || - CreateMaterializedViewStmt.class.isAssignableFrom(statementClazz) || - CreateTableAsSelectStmt.class.isAssignableFrom(statementClazz))) { + if (statementClazz != null && !QueryStmt.class.isAssignableFrom(statementClazz)) { columnLabel = item.toColumnLabel(i); } if (columnLabel == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java index 015edb2fe50307..3cac20419711f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java @@ -84,6 +84,9 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { return; } this.analyzer = analyzer; + if (analyzer.getStatementClazz() == null) { + analyzer.setStatementClazz(this.getClass()); + } if (Strings.isNullOrEmpty(analyzer.getClusterName())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_SELECT_CLUSTER); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index f243fb30e6290d..2ce0de22cb2d6c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -719,7 +719,6 @@ public void executeByLegacy(TUniqueId queryId) throws Exception { } } else { analyzer = new Analyzer(context.getEnv(), context); - analyzer.setStatementClazz(parsedStmt.getClass()); parsedStmt.analyze(analyzer); } @@ -934,7 +933,6 @@ public void analyze(TQueryOptions tQueryOptions) throws UserException, Interrupt } analyzer = new Analyzer(context.getEnv(), context); - analyzer.setStatementClazz(parsedStmt.getClass()); if (parsedStmt instanceof PrepareStmt || context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java index 934521a094ba59..b5e3e0e74a1f56 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java @@ -18,7 +18,9 @@ package org.apache.doris.qe; import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateViewStmt; import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.QueryStmt; import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.SqlScanner; import org.apache.doris.analysis.StatementBase; @@ -165,22 +167,6 @@ public SystemInfoService getCurrentSystemInfo() { }; db = new Database(1L, fullDbName); - OlapTable tbl1 = createOrderTable(); - OlapTable tbl2 = createProfileTable(); - OlapTable tbl3 = createEventTable(); - db.createTable(tbl1); - db.createTable(tbl2); - db.createTable(tbl3); - - View view1 = createEventView1(); - View view2 = createEventView2(); - View view3 = createEventView3(); - View view4 = createEventNestedView(); - db.createTable(view1); - db.createTable(view2); - db.createTable(view3); - db.createTable(view4); - new Expectations(catalog) { { catalog.getDbNullable(fullDbName); @@ -334,6 +320,25 @@ public SystemInfoService getCurrentSystemInfo() { analyzer = new Analyzer(env, ctx); newRangeList = Lists.newArrayList(); + + // table and view init use analyzer, should init after analyzer build + OlapTable tbl1 = createOrderTable(); + db.createTable(tbl1); + OlapTable tbl2 = createProfileTable(); + db.createTable(tbl2); + OlapTable tbl3 = createEventTable(); + db.createTable(tbl3); + + // build view meta inline sql and create view directly, the originStmt from inline sql + // should be analyzed by create view statement analyzer and then to sql + View view1 = createEventView1(); + db.createTable(view1); + View view2 = createEventView2(); + db.createTable(view2); + View view3 = createEventView3(); + db.createTable(view3); + View view4 = createEventNestedView(); + db.createTable(view4); } private OlapTable createOrderTable() { @@ -509,14 +514,18 @@ private View createEventView1() { String originStmt = "select eventdate, COUNT(userid) FROM appevent WHERE " + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"; View view = new View(30000L, "view1", null); - view.setInlineViewDefWithSqlMode(originStmt, 0L); + Analyzer createViewAnalyzer = new Analyzer(env, ctx); + createViewAnalyzer.setStatementClazz(CreateViewStmt.class); + view.setInlineViewDefWithSqlMode(parseSql(originStmt, createViewAnalyzer, true).toSql(), 0L); return view; } private View createEventView2() { String originStmt = "select eventdate, userid FROM appevent"; View view = new View(30001L, "view2", null); - view.setInlineViewDefWithSqlMode(originStmt, 0L); + Analyzer createViewAnalyzer = new Analyzer(env, ctx); + createViewAnalyzer.setStatementClazz(CreateViewStmt.class); + view.setInlineViewDefWithSqlMode(parseSql(originStmt, createViewAnalyzer, true).toSql(), 0L); return view; } @@ -524,7 +533,9 @@ private View createEventView3() { String originStmt = "select eventdate, COUNT(userid) FROM appevent WHERE " + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate"; View view = new View(30002L, "view3", null); - view.setInlineViewDefWithSqlMode(originStmt, 0L); + Analyzer createViewAnalyzer = new Analyzer(env, ctx); + createViewAnalyzer.setStatementClazz(CreateViewStmt.class); + view.setInlineViewDefWithSqlMode(parseSql(originStmt, createViewAnalyzer, true).toSql(), 0L); return view; } @@ -532,7 +543,10 @@ private View createEventNestedView() { String originStmt = "select eventdate, COUNT(userid) FROM view2 WHERE " + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"; View view = new View(30003L, "view4", null); - view.setInlineViewDefWithSqlMode(originStmt, 0L); + Analyzer createViewAnalyzer = new Analyzer(env, ctx); + createViewAnalyzer.setStatementClazz(CreateViewStmt.class); + view.setInlineViewDefWithSqlMode( + parseSql(originStmt, createViewAnalyzer, true).toSql(), 0L); return view; } @@ -564,11 +578,18 @@ private StatementBase parseSqlByNereids(String sql) { } private StatementBase parseSql(String sql) { + return parseSql(sql, null, false); + } + + private StatementBase parseSql(String sql, Analyzer analyzer, boolean needToSql) { SqlParser parser = new SqlParser(new SqlScanner(new StringReader(sql))); StatementBase parseStmt = null; try { parseStmt = SqlParserUtils.getFirstStmt(parser); - parseStmt.analyze(analyzer); + if (parseStmt instanceof QueryStmt) { + ((QueryStmt) parseStmt).setNeedToSql(needToSql); + } + parseStmt.analyze(analyzer == null ? this.analyzer : analyzer); } catch (AnalysisException e) { LOG.warn("Part,an_ex={}", e); Assert.fail(e.getMessage()); @@ -1023,10 +1044,10 @@ public void testSubSelect() throws Exception { cache.rewriteSelectStmt(null); LOG.warn("Sub nokey={}", cache.getNokeyStmt().toSql()); Assert.assertEquals(cache.getNokeyStmt().toSql(), - "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `__sum_1` FROM (" - + "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `pv` FROM " - + "`testCluster:testDb`.`appevent` WHERE `eventid` = 1" - + " GROUP BY `eventdate`) tbl GROUP BY `eventdate`"); + "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` " + + "FROM (SELECT `eventdate` AS `eventdate`, count(`userid`) AS `pv` " + + "FROM `testCluster:testDb`.`appevent` WHERE `eventid` = 1 GROUP BY `eventdate`) tbl " + + "GROUP BY `eventdate`"); PartitionRange range = cache.getPartitionRange(); boolean flag = range.analytics(); @@ -1045,11 +1066,11 @@ public void testSubSelect() throws Exception { sql = ca.getRewriteStmt().toSql(); LOG.warn("Sub rewrite={}", sql); Assert.assertEquals(sql, - "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `__sum_1` FROM (" - + "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `pv` FROM " - + "`testCluster:testDb`.`appevent` WHERE " - + "`eventdate` > '2020-01-13' AND `eventdate` < '2020-01-16' AND `eventid` = 1 GROUP BY " - + "`eventdate`) tbl GROUP BY `eventdate`"); + "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` " + + "FROM (SELECT `eventdate` AS `eventdate`, count(`userid`) AS `pv` " + + "FROM `testCluster:testDb`.`appevent` WHERE `eventdate` > '2020-01-13' AND " + + "`eventdate` < '2020-01-16' AND `eventid` = 1 GROUP BY `eventdate`) tbl " + + "GROUP BY `eventdate`"); } catch (Exception e) { LOG.warn("sub ex={}", e); Assert.fail(e.getMessage()); @@ -1101,9 +1122,9 @@ public void testSqlCacheKey() { SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); - Assert.assertEquals(cacheKey, "SELECT `eventdate` AS `eventdate`, count(`userid`) AS " - + "`__count_1` FROM `testCluster:testDb`.`appevent` WHERE `eventdate` >= '2020-01-12' AND " - + "`eventdate` <= '2020-01-14' GROUP BY `eventdate`|"); + Assert.assertEquals(cacheKey, "SELECT `eventdate` AS `eventdate`, " + + " count(`userid`) AS `count(``userid``)` FROM `testCluster:testDb`.`appevent` " + + "WHERE `eventdate` >= '2020-01-12' AND `eventdate` <= '2020-01-14' GROUP BY `eventdate`|"); Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum()); } @@ -1140,9 +1161,11 @@ public void testSqlCacheKeyWithView() { SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); - Assert.assertEquals(cacheKey, "SELECT `testCluster:testDb`.`view1`.`eventdate` AS `eventdate`, `testCluster:testDb`.`view1`." - + "`__count_1` AS `__count_1` FROM `testCluster:testDb`.`view1`|select eventdate, COUNT(userid) " - + "FROM appevent WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"); + Assert.assertEquals(cacheKey, "SELECT `testCluster:testDb`.`view1`.`eventdate` AS `eventdate`, " + + "`testCluster:testDb`.`view1`.`__count_1` AS `__count_1` FROM `testCluster:testDb`.`view1`|" + + "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `__count_1` FROM " + + "`testCluster:testDb`.`appevent` WHERE `eventdate` >= '2020-01-12' AND " + + "`eventdate` <= '2020-01-14' GROUP BY `eventdate`"); Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum()); } @@ -1238,10 +1261,11 @@ public void testPartitionCacheKeyWithView() { cache.rewriteSelectStmt(null); Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); - Assert.assertEquals(cache.getSqlWithViewStmt(), "SELECT `testCluster:testDb`.`view3`.`eventdate` AS " - + "`eventdate`, `testCluster:testDb`.`view3`.`__count_1` AS `__count_1` FROM " - + "`testCluster:testDb`.`view3`|select eventdate, COUNT(userid) FROM appevent WHERE eventdate>=" - + "\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate"); + Assert.assertEquals(cache.getSqlWithViewStmt(), "SELECT `testCluster:testDb`.`view3`.`eventdate` " + + "AS `eventdate`, `testCluster:testDb`.`view3`.`__count_1` AS `__count_1` " + + "FROM `testCluster:testDb`.`view3`|SELECT `eventdate` AS `eventdate`, count(`userid`) " + + "AS `__count_1` FROM `testCluster:testDb`.`appevent` WHERE `eventdate` >= '2020-01-12' " + + "AND `eventdate` <= '2020-01-15' GROUP BY `eventdate`"); } catch (Exception e) { LOG.warn("ex={}", e); Assert.fail(e.getMessage()); @@ -1295,9 +1319,10 @@ public void testSqlCacheKeyWithNestedView() { SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); Assert.assertEquals(cacheKey, "SELECT `testCluster:testDb`.`view4`.`eventdate` AS `eventdate`, " - + "`testCluster:testDb`.`view4`.`__count_1` AS `__count_1` FROM `testCluster:testDb`.`view4`|select " - + "eventdate, COUNT(userid) FROM view2 WHERE eventdate>=\"2020-01-12\" and " - + "eventdate<=\"2020-01-14\" GROUP BY eventdate|select eventdate, userid FROM appevent"); + + "`testCluster:testDb`.`view4`.`__count_1` AS `__count_1` FROM `testCluster:testDb`.`view4`|" + + "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `__count_1` FROM `testDb`.`view2` " + + "WHERE `eventdate` >= '2020-01-12' AND `eventdate` <= '2020-01-14' GROUP BY `eventdate`|" + + "SELECT `eventdate` AS `eventdate`, `userid` AS `userid` FROM `testCluster:testDb`.`appevent`"); Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum()); } From f7ae013959cec1280030afbed51ebf1c0a7aab74 Mon Sep 17 00:00:00 2001 From: JingDas Date: Thu, 12 Oct 2023 20:02:58 +0800 Subject: [PATCH 4/7] [fix](meta) fix ut test fail --- .../analysis/AlterMaterializedViewStmt.java | 4 +- .../analysis/CreateTableAsSelectStmt.java | 1 + .../apache/doris/analysis/CreateViewStmt.java | 1 + .../apache/doris/analysis/StatementBase.java | 1 + .../apache/doris/qe/OlapQueryCacheTest.java | 40 ++++++++++--------- 5 files changed, 27 insertions(+), 20 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterMaterializedViewStmt.java index a59ed5b74dc1d0..b0ecc5261a65e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterMaterializedViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterMaterializedViewStmt.java @@ -21,6 +21,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; @@ -42,7 +43,8 @@ public MVRefreshInfo getRefreshInfo() { } @Override - public void analyze(Analyzer analyzer) throws AnalysisException { + public void analyze(Analyzer analyzer) throws AnalysisException, UserException { + super.analyze(analyzer); mvName.analyze(analyzer); if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(), mvName.getDb(), mvName.getTbl(), PrivPredicate.ALTER)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java index 3019117942ac92..fc737b0340cd09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java @@ -71,6 +71,7 @@ public void analyze(Analyzer analyzer) throws UserException { // first: we analyze queryStmt before create table. // To avoid duplicate registrations of table/colRefs, // create a new root analyzer and clone the query statement for this initial pass. + super.analyze(analyzer); Analyzer dummyRootAnalyzer = new Analyzer(analyzer.getEnv(), analyzer.getContext()); QueryStmt tmpStmt = queryStmt.clone(); tmpStmt.analyze(dummyRootAnalyzer); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java index 528b882f848991..befbe8a9dc94da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateViewStmt.java @@ -55,6 +55,7 @@ public String getComment() { @Override public void analyze(Analyzer analyzer) throws UserException { + super.analyze(analyzer); tableName.analyze(analyzer); FeNameFormat.checkTableName(tableName.getTbl()); viewDefStmt.setNeedToSql(true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java index 3cac20419711f5..0ef354e334e69a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java @@ -78,6 +78,7 @@ protected StatementBase(StatementBase other) { * were missing from the catalog. * It is up to the analysis() implementation to ensure the maximum number of missing * tables/views get collected in the Analyzer before failing analyze(). + * Should call the method firstly when override the method. */ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { if (isAnalyzed()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java index b5e3e0e74a1f56..7b6958346c3846 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java @@ -1182,9 +1182,9 @@ public void testSqlCacheKeyWithViewForNereids() { SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); - Assert.assertEquals(cacheKey, "SELECT * from testDb.view1" - + "|select eventdate, COUNT(userid) FROM appevent " - + "WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"); + Assert.assertEquals(cacheKey, "SELECT * from testDb.view1|SELECT `eventdate` AS `eventdate`, " + + "count(`userid`) AS `__count_1` FROM `testCluster:testDb`.`appevent` " + + "WHERE `eventdate` >= '2020-01-12' AND `eventdate` <= '2020-01-14' GROUP BY `eventdate`"); Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum()); } @@ -1208,10 +1208,11 @@ public void testSqlCacheKeyWithSubSelectView() { SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); - Assert.assertEquals(cacheKey, "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`userid` AS " - + "`userid` FROM (SELECT `view2`.`eventdate` AS `eventdate`, `view2`.`userid` AS `userid` FROM " - + "`testCluster:testDb`.`view2` view2 WHERE `view2`.`eventdate` >= '2020-01-12' AND `view2`.`eventdate` " - + "<= '2020-01-14') origin|select eventdate, userid FROM appevent"); + Assert.assertEquals(cacheKey, "SELECT `origin`.`eventdate` AS `eventdate`, " + + "`origin`.`userid` AS `userid` FROM (SELECT `view2`.`eventdate` AS `eventdate`, " + + "`view2`.`userid` AS `userid` FROM `testCluster:testDb`.`view2` view2 " + + "WHERE `view2`.`eventdate` >= '2020-01-12' AND `view2`.`eventdate` <= '2020-01-14') origin|" + + "SELECT `eventdate` AS `eventdate`, `userid` AS `userid` FROM `testCluster:testDb`.`appevent`"); Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum()); } @@ -1237,11 +1238,11 @@ public void testSqlCacheKeyWithSubSelectViewForNereids() { SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); Assert.assertEquals(cacheKey, "select origin.eventdate as eventdate, origin.userid as userid\n" - + "from (\n" - + " select view2.eventdate as eventdate, view2.userid as userid \n" - + " from testDb.view2 view2 \n" - + " where view2.eventdate >=\"2020-01-12\" and view2.eventdate <= \"2020-01-14\"\n" - + ") origin" + "|select eventdate, userid FROM appevent"); + + "from (\n" + + " select view2.eventdate as eventdate, view2.userid as userid \n" + + " from testDb.view2 view2 \n" + + " where view2.eventdate >=\"2020-01-12\" and view2.eventdate <= \"2020-01-14\"\n" + + ") origin|SELECT `eventdate` AS `eventdate`, `userid` AS `userid` FROM `testCluster:testDb`.`appevent`"); Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum()); } @@ -1296,9 +1297,10 @@ public void testPartitionCacheKeyWithSubSelectView() { cache.rewriteSelectStmt(null); Assert.assertEquals(cache.getNokeyStmt().getWhereClause(), null); Assert.assertEquals(cache.getSqlWithViewStmt(), - "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`cnt` AS `cnt` FROM (SELECT " - + " `eventdate` AS `eventdate`, count(`userid`) AS `cnt` FROM " - + "`testDb`.`view2` GROUP BY `eventdate`) origin|select eventdate, userid FROM appevent"); + "SELECT `origin`.`eventdate` AS `eventdate`, `origin`.`cnt` AS `cnt` " + + "FROM (SELECT `eventdate` AS `eventdate`, count(`userid`) AS `cnt` " + + "FROM `testDb`.`view2` GROUP BY `eventdate`) origin|SELECT `eventdate` AS `eventdate`, " + + "`userid` AS `userid` FROM `testCluster:testDb`.`appevent`"); } catch (Exception e) { LOG.warn("ex={}", e); Assert.fail(e.getMessage()); @@ -1339,10 +1341,10 @@ public void testSqlCacheKeyWithNestedViewForNereids() { SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); - Assert.assertEquals(cacheKey, "SELECT * from testDb.view4" - + "|select eventdate, COUNT(userid) FROM view2 " - + "WHERE eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate" - + "|select eventdate, userid FROM appevent"); + Assert.assertEquals(cacheKey, "SELECT * from testDb.view4|SELECT `eventdate` AS `eventdate`, " + + "count(`userid`) AS `__count_1` FROM `testDb`.`view2` WHERE `eventdate` >= '2020-01-12' AND " + + "`eventdate` <= '2020-01-14' GROUP BY `eventdate`|SELECT `eventdate` AS `eventdate`, " + + "`userid` AS `userid` FROM `testCluster:testDb`.`appevent`"); Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum()); } From 48ff6e475c7e256f2d1e44e4fa62af77de722ca7 Mon Sep 17 00:00:00 2001 From: JingDas Date: Thu, 12 Oct 2023 20:38:33 +0800 Subject: [PATCH 5/7] [fix](meta) fix ut test fail when create table with select --- .../org/apache/doris/analysis/CreateTableAsSelectStmt.java | 2 +- .../src/main/java/org/apache/doris/analysis/SelectStmt.java | 4 +++- .../main/java/org/apache/doris/analysis/StatementBase.java | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java index fc737b0340cd09..f2bca083b3b7db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableAsSelectStmt.java @@ -71,8 +71,8 @@ public void analyze(Analyzer analyzer) throws UserException { // first: we analyze queryStmt before create table. // To avoid duplicate registrations of table/colRefs, // create a new root analyzer and clone the query statement for this initial pass. - super.analyze(analyzer); Analyzer dummyRootAnalyzer = new Analyzer(analyzer.getEnv(), analyzer.getContext()); + super.analyze(dummyRootAnalyzer); QueryStmt tmpStmt = queryStmt.clone(); tmpStmt.analyze(dummyRootAnalyzer); this.queryStmt = tmpStmt; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index eeb0334e6db7f4..ee6a5ccb65e78c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -546,10 +546,12 @@ public void analyze(Analyzer analyzer) throws UserException { String columnLabel = null; Class statementClazz = analyzer.getStatementClazz(); if (statementClazz != null && !QueryStmt.class.isAssignableFrom(statementClazz)) { + // Infer column name when item is expr columnLabel = item.toColumnLabel(i); } if (columnLabel == null) { - // column label without position is applicative for query + // column label without position is applicative for query and do not infer + // column name when item is expr columnLabel = item.toColumnLabel(); } SlotRef aliasRef = new SlotRef(null, columnLabel); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java index 0ef354e334e69a..6071a1e02e090f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java @@ -78,7 +78,8 @@ protected StatementBase(StatementBase other) { * were missing from the catalog. * It is up to the analysis() implementation to ensure the maximum number of missing * tables/views get collected in the Analyzer before failing analyze(). - * Should call the method firstly when override the method. + * Should call the method firstly when override the method, the analyzer param should be + * the one which statement would use. */ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { if (isAnalyzed()) { From 3c8010bf4f486d181f22910bdc643bd5cf7a8582 Mon Sep 17 00:00:00 2001 From: JingDas Date: Thu, 12 Oct 2023 22:37:45 +0800 Subject: [PATCH 6/7] [fix](meta) optimize ut test --- .../apache/doris/qe/OlapQueryCacheTest.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java index 7b6958346c3846..cd6201e4015a04 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java @@ -1044,10 +1044,10 @@ public void testSubSelect() throws Exception { cache.rewriteSelectStmt(null); LOG.warn("Sub nokey={}", cache.getNokeyStmt().toSql()); Assert.assertEquals(cache.getNokeyStmt().toSql(), - "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` " - + "FROM (SELECT `eventdate` AS `eventdate`, count(`userid`) AS `pv` " - + "FROM `testCluster:testDb`.`appevent` WHERE `eventid` = 1 GROUP BY `eventdate`) tbl " - + "GROUP BY `eventdate`"); + "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` FROM (" + + "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `pv` FROM " + + "`testCluster:testDb`.`appevent` WHERE `eventid` = 1" + + " GROUP BY `eventdate`) tbl GROUP BY `eventdate`"); PartitionRange range = cache.getPartitionRange(); boolean flag = range.analytics(); @@ -1066,11 +1066,11 @@ public void testSubSelect() throws Exception { sql = ca.getRewriteStmt().toSql(); LOG.warn("Sub rewrite={}", sql); Assert.assertEquals(sql, - "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` " - + "FROM (SELECT `eventdate` AS `eventdate`, count(`userid`) AS `pv` " - + "FROM `testCluster:testDb`.`appevent` WHERE `eventdate` > '2020-01-13' AND " - + "`eventdate` < '2020-01-16' AND `eventid` = 1 GROUP BY `eventdate`) tbl " - + "GROUP BY `eventdate`"); + "SELECT `eventdate` AS `eventdate`, sum(`pv`) AS `sum(``pv``)` FROM (" + + "SELECT `eventdate` AS `eventdate`, count(`userid`) AS `pv` FROM " + + "`testCluster:testDb`.`appevent` WHERE " + + "`eventdate` > '2020-01-13' AND `eventdate` < '2020-01-16' AND `eventid` = 1 GROUP BY " + + "`eventdate`) tbl GROUP BY `eventdate`"); } catch (Exception e) { LOG.warn("sub ex={}", e); Assert.fail(e.getMessage()); @@ -1122,9 +1122,9 @@ public void testSqlCacheKey() { SqlCache sqlCache = (SqlCache) ca.getCache(); String cacheKey = sqlCache.getSqlWithViewStmt(); - Assert.assertEquals(cacheKey, "SELECT `eventdate` AS `eventdate`, " - + " count(`userid`) AS `count(``userid``)` FROM `testCluster:testDb`.`appevent` " - + "WHERE `eventdate` >= '2020-01-12' AND `eventdate` <= '2020-01-14' GROUP BY `eventdate`|"); + Assert.assertEquals(cacheKey, "SELECT `eventdate` AS `eventdate`, count(`userid`) AS " + + "`count(``userid``)` FROM `testCluster:testDb`.`appevent` WHERE `eventdate` >= '2020-01-12' AND " + + "`eventdate` <= '2020-01-14' GROUP BY `eventdate`|"); Assert.assertEquals(selectedPartitionIds.size(), sqlCache.getSumOfPartitionNum()); } From 27da0303fa7216f2e62475beb248f61d637e689e Mon Sep 17 00:00:00 2001 From: JingDas Date: Fri, 13 Oct 2023 10:48:41 +0800 Subject: [PATCH 7/7] [fix](meta) rename the field --- .../main/java/org/apache/doris/analysis/Analyzer.java | 10 +++++----- .../java/org/apache/doris/analysis/SelectStmt.java | 2 +- .../java/org/apache/doris/analysis/StatementBase.java | 4 ++-- .../java/org/apache/doris/qe/OlapQueryCacheTest.java | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java index 9571ef505b88a4..07c483d7685b34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java @@ -321,7 +321,7 @@ private static class GlobalState { // Record the statement clazz that the analyzer would to analyze, // this give an opportunity to control analyzing behavior according to the statement type. - public Class statementClazz; + public Class rootStatementClazz; // Indicates whether the query has plan hints. public boolean hasPlanHints = false; @@ -571,12 +571,12 @@ public boolean isExplain() { return globalState.isExplain; } - public void setStatementClazz(Class statementClazz) { - globalState.statementClazz = statementClazz; + public void setRootStatementClazz(Class statementClazz) { + globalState.rootStatementClazz = statementClazz; } - public Class getStatementClazz() { - return globalState.statementClazz; + public Class getRootStatementClazz() { + return globalState.rootStatementClazz; } public int incrementCallDepth() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index ee6a5ccb65e78c..8c40f5230ac116 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -544,7 +544,7 @@ public void analyze(Analyzer analyzer) throws UserException { } resultExprs.add(rewriteQueryExprByMvColumnExpr(item.getExpr(), analyzer)); String columnLabel = null; - Class statementClazz = analyzer.getStatementClazz(); + Class statementClazz = analyzer.getRootStatementClazz(); if (statementClazz != null && !QueryStmt.class.isAssignableFrom(statementClazz)) { // Infer column name when item is expr columnLabel = item.toColumnLabel(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java index 6071a1e02e090f..13895f5695bbd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java @@ -86,8 +86,8 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { return; } this.analyzer = analyzer; - if (analyzer.getStatementClazz() == null) { - analyzer.setStatementClazz(this.getClass()); + if (analyzer.getRootStatementClazz() == null) { + analyzer.setRootStatementClazz(this.getClass()); } if (Strings.isNullOrEmpty(analyzer.getClusterName())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_SELECT_CLUSTER); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java index cd6201e4015a04..5e2d13003f2da5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java @@ -515,7 +515,7 @@ private View createEventView1() { + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"; View view = new View(30000L, "view1", null); Analyzer createViewAnalyzer = new Analyzer(env, ctx); - createViewAnalyzer.setStatementClazz(CreateViewStmt.class); + createViewAnalyzer.setRootStatementClazz(CreateViewStmt.class); view.setInlineViewDefWithSqlMode(parseSql(originStmt, createViewAnalyzer, true).toSql(), 0L); return view; } @@ -524,7 +524,7 @@ private View createEventView2() { String originStmt = "select eventdate, userid FROM appevent"; View view = new View(30001L, "view2", null); Analyzer createViewAnalyzer = new Analyzer(env, ctx); - createViewAnalyzer.setStatementClazz(CreateViewStmt.class); + createViewAnalyzer.setRootStatementClazz(CreateViewStmt.class); view.setInlineViewDefWithSqlMode(parseSql(originStmt, createViewAnalyzer, true).toSql(), 0L); return view; } @@ -534,7 +534,7 @@ private View createEventView3() { + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-15\" GROUP BY eventdate"; View view = new View(30002L, "view3", null); Analyzer createViewAnalyzer = new Analyzer(env, ctx); - createViewAnalyzer.setStatementClazz(CreateViewStmt.class); + createViewAnalyzer.setRootStatementClazz(CreateViewStmt.class); view.setInlineViewDefWithSqlMode(parseSql(originStmt, createViewAnalyzer, true).toSql(), 0L); return view; } @@ -544,7 +544,7 @@ private View createEventNestedView() { + "eventdate>=\"2020-01-12\" and eventdate<=\"2020-01-14\" GROUP BY eventdate"; View view = new View(30003L, "view4", null); Analyzer createViewAnalyzer = new Analyzer(env, ctx); - createViewAnalyzer.setStatementClazz(CreateViewStmt.class); + createViewAnalyzer.setRootStatementClazz(CreateViewStmt.class); view.setInlineViewDefWithSqlMode( parseSql(originStmt, createViewAnalyzer, true).toSql(), 0L); return view;