From 98b149d07fa285f095ce2592db87e26b5c7b316f Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Wed, 28 Aug 2024 16:57:05 +0800 Subject: [PATCH 1/2] support sql cache between different comment --- .../doris/common/NereidsSqlCacheManager.java | 11 +++++-- .../doris/nereids/parser/NereidsParser.java | 32 +++++++++++++++++++ .../doris/regression/util/LoggerUtils.groovy | 29 ++++++++++++++--- regression-test/plugins/test_helper.groovy | 2 ++ .../cache/parse_sql_from_sql_cache.groovy | 20 +++++++++--- 5 files changed, 82 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index ba2c978830a489..1317fdaefc766e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -38,6 +38,7 @@ import org.apache.doris.nereids.SqlCacheContext.ScanTable; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundVariable; +import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer; import org.apache.doris.nereids.rules.analysis.UserAuthentication; @@ -128,7 +129,7 @@ public void tryAddFeSqlCache(ConnectContext connectContext, String sql) { SqlCacheContext sqlCacheContext = sqlCacheContextOpt.get(); UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity(); String key = sqlCacheContext.getCacheKeyType() == CacheKeyType.SQL - ? currentUserIdentity.toString() + ":" + sql.trim() + ? currentUserIdentity.toString() + ":" + normalizeSql(sql.trim()) : currentUserIdentity.toString() + ":" + DebugUtil.printId(sqlCacheContext.getOrComputeCacheKeyMd5()); if (sqlCaches.getIfPresent(key) == null && sqlCacheContext.getOrComputeCacheKeyMd5() != null && sqlCacheContext.getResultSetInFe().isPresent()) { @@ -148,7 +149,7 @@ public void tryAddBeCache(ConnectContext connectContext, String sql, CacheAnalyz SqlCacheContext sqlCacheContext = sqlCacheContextOpt.get(); UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity(); String key = sqlCacheContext.getCacheKeyType() == CacheKeyType.SQL - ? currentUserIdentity.toString() + ":" + sql.trim() + ? currentUserIdentity.toString() + ":" + normalizeSql(sql.trim()) : currentUserIdentity.toString() + ":" + DebugUtil.printId(sqlCacheContext.getOrComputeCacheKeyMd5()); if (sqlCaches.getIfPresent(key) == null && sqlCacheContext.getOrComputeCacheKeyMd5() != null) { SqlCache cache = (SqlCache) analyzer.getCache(); @@ -169,7 +170,7 @@ public void tryAddBeCache(ConnectContext connectContext, String sql, CacheAnalyz /** tryParseSql */ public Optional tryParseSql(ConnectContext connectContext, String sql) { UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity(); - String key = currentUserIdentity + ":" + sql.trim(); + String key = currentUserIdentity + ":" + normalizeSql(sql.trim()); SqlCacheContext sqlCacheContext = sqlCaches.getIfPresent(key); if (sqlCacheContext == null) { return Optional.empty(); @@ -202,6 +203,10 @@ public Optional tryParseSql(ConnectContext connectContext, Stri } } + private String normalizeSql(String sql) { + return NereidsParser.removeCommentAndTrimBlank(sql); + } + private Optional tryParseSqlWithoutCheckVariable( ConnectContext connectContext, String key, SqlCacheContext sqlCacheContext, UserIdentity currentUserIdentity) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index f6f05392cb81b7..7cefe34c9fcc8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -40,6 +40,7 @@ import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.TokenSource; import org.antlr.v4.runtime.atn.PredictionMode; @@ -342,4 +343,35 @@ public static ParserRuleContext toAst(String sql, Function getErrorInfo(Throwable t, File file) { if (file.name.endsWith(".groovy")) { + def st = findRootErrorStackTrace(t, Sets.newLinkedHashSet(), file) int lineNumber = -1 - for (def st : t.getStackTrace()) { - if (Objects.equals(st.fileName, file.name)) { - lineNumber = st.getLineNumber() - break - } + if (!st.is(null)) { + lineNumber = st.getLineNumber() } if (lineNumber == -1) { return new Tuple2(null, null) @@ -40,4 +40,23 @@ class LoggerUtils { return new Tuple2(null, null) } } + + static StackTraceElement findRootErrorStackTrace(Throwable t, Set throwables, File file) { + throwables.add(t) + + def cause = t.getCause() + if (!cause.is(null) && !throwables.contains(cause)) { + def foundStackTrace = findRootErrorStackTrace(cause, throwables, file) + if (!foundStackTrace.is(null)) { + return foundStackTrace + } + } + + for (def st : t.getStackTrace()) { + if (Objects.equals(st.fileName, file.name)) { + return st + } + } + return null + } } diff --git a/regression-test/plugins/test_helper.groovy b/regression-test/plugins/test_helper.groovy index 4f7eeb3c09b380..60fb9d2c8ed40f 100644 --- a/regression-test/plugins/test_helper.groovy +++ b/regression-test/plugins/test_helper.groovy @@ -57,6 +57,8 @@ Suite.metaClass.createTestTable = { String tableName, boolean uniqueTable = fals (4, 1), (4, 2), (5, 1), (5, 2) """ + + sql "sync" } diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy index 2b60f98514a4c3..3381b5653c4865 100644 --- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy @@ -722,10 +722,7 @@ suite("parse_sql_from_sql_cache") { def result2 = sql "select * from (select $randomInt as id)a" assertTrue(result2.size() == 1) - assertNoCache "select * from test_use_plan_cache20 limit 0" - def result3 = sql "select * from test_use_plan_cache20 limit 0" - assertTrue(result3.isEmpty()) - + sql "select * from test_use_plan_cache20 limit 0" assertHasCache "select * from test_use_plan_cache20 limit 0" def result4 = sql "select * from test_use_plan_cache20 limit 0" assertTrue(result4.isEmpty()) @@ -771,6 +768,21 @@ suite("parse_sql_from_sql_cache") { assertNoCache "select * from test_use_plan_cache21" def result2 = sql "select * from test_use_plan_cache21" assertTrue(result2.size() == 1) + }), + extraThread("remove_comment", { + createTestTable "test_use_plan_cache22" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment2*/ * from test_use_plan_cache22 order by 1, 2" + sql "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment1*/ * from test_use_plan_cache22 order by 1, 2" + + assertHasCache "select /*+SET_VAR(disable_nereids_rules='')*/ /*comment2*/ * from test_use_plan_cache22 order by 1, 2" }) ).get() } From ced14448e64f750045d73759c5b24668e0433c10 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Wed, 28 Aug 2024 17:05:13 +0800 Subject: [PATCH 2/2] fix --- .../org/apache/doris/nereids/parser/NereidsParser.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 7cefe34c9fcc8b..34646c1d657953 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -344,6 +344,13 @@ public static ParserRuleContext toAst(String sql, Function