From 8fdaf9867136dd1084d6c887e7080d5284ab02d9 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 11 Mar 2025 12:14:31 +0800 Subject: [PATCH 1/4] fix --- .../doris/nereids/StatementContext.java | 2 +- .../cache/parse_sql_from_sql_cache.groovy | 7 +--- .../cache/prepare_stmt_with_sql_cache.groovy | 37 ++++++++++++++----- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 75353f446a4433..e1f290999f1f92 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -166,7 +166,7 @@ public enum TableFrom { private final Stack plannerResources = new Stack<>(); // placeholder params for prepared statement - private List placeholders; + private List placeholders = new ArrayList<>(); // all tables in query private boolean needLockTables = true; diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy index 765d1208426607..1542efe984a27a 100644 --- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy @@ -33,12 +33,7 @@ suite("parse_sql_from_sql_cache") { } def dbName = (sql "select database()")[0][0].toString() - foreachFrontends { fe -> - def url = "jdbc:mysql://${fe.Host}:${fe.QueryPort}/${dbName}" - connect(context.config.jdbcUser, context.config.jdbcPassword, url) { - sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' = '10')" - } - } + sql "ADMIN SET ALL FRONTENDS CONFIG ('cache_last_version_interval_second' = '10')" // make sure if the table has been dropped, the cache should invalidate, // so we should retry multiple times to check diff --git a/regression-test/suites/nereids_p0/cache/prepare_stmt_with_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/prepare_stmt_with_sql_cache.groovy index 7819a6ca09d719..442a82a2817a34 100644 --- a/regression-test/suites/nereids_p0/cache/prepare_stmt_with_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/prepare_stmt_with_sql_cache.groovy @@ -15,17 +15,10 @@ // specific language governing permissions and limitations // under the License. -import com.mysql.cj.ServerPreparedQuery -import com.mysql.cj.jdbc.ConnectionImpl -import com.mysql.cj.jdbc.JdbcStatement -import com.mysql.cj.jdbc.ServerPreparedStatement -import com.mysql.cj.jdbc.StatementImpl import org.apache.doris.regression.util.JdbcUtils -import java.lang.reflect.Field import java.sql.PreparedStatement import java.sql.ResultSet -import java.util.concurrent.CopyOnWriteArrayList suite("prepare_stmt_with_sql_cache") { @@ -38,11 +31,13 @@ suite("prepare_stmt_with_sql_cache") { insert into test_prepare_stmt_with_sql_cache select * from numbers('number'='100'); """ + sql "ADMIN SET ALL FRONTENDS CONFIG ('cache_last_version_interval_second' = '10')" + def db = (sql "select database()")[0][0].toString() - def url = getServerPrepareJdbcUrl(context.config.jdbcUrl, db) + def serverPrepareUrl = getServerPrepareJdbcUrl(context.config.jdbcUrl, db) - connect(context.config.jdbcUser, context.config.jdbcPassword, url) { + connect(context.config.jdbcUser, context.config.jdbcPassword, serverPrepareUrl) { sql "set enable_sql_cache=true" for (def i in 0..<10) { try (PreparedStatement pstmt = prepareStatement("select * from test_prepare_stmt_with_sql_cache where id=?")) { @@ -54,4 +49,28 @@ suite("prepare_stmt_with_sql_cache") { } } } + + sleep(10 * 1000) + + connect(context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) { + sql "use ${db}" + sql "set enable_sql_cache=true" + test { + sql "select * from test_prepare_stmt_with_sql_cache where id=10" + result([[10]]) + } + test { + sql "select * from test_prepare_stmt_with_sql_cache where id=10" + result([[10]]) + } + } + + connect(context.config.jdbcUser, context.config.jdbcPassword, serverPrepareUrl) { + sql "use ${db}" + sql "set enable_sql_cache=true" + test { + sql "select * from test_prepare_stmt_with_sql_cache where id=10" + result(([[10]])) + } + } } From 991e98d74115caa996a5296a25ca872e4cad73fc Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 11 Mar 2025 12:18:45 +0800 Subject: [PATCH 2/4] fix --- .../nereids_p0/cache/prepare_stmt_with_sql_cache.groovy | 4 ---- 1 file changed, 4 deletions(-) diff --git a/regression-test/suites/nereids_p0/cache/prepare_stmt_with_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/prepare_stmt_with_sql_cache.groovy index 442a82a2817a34..1d358adfbff803 100644 --- a/regression-test/suites/nereids_p0/cache/prepare_stmt_with_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/prepare_stmt_with_sql_cache.groovy @@ -59,10 +59,6 @@ suite("prepare_stmt_with_sql_cache") { sql "select * from test_prepare_stmt_with_sql_cache where id=10" result([[10]]) } - test { - sql "select * from test_prepare_stmt_with_sql_cache where id=10" - result([[10]]) - } } connect(context.config.jdbcUser, context.config.jdbcPassword, serverPrepareUrl) { From 1657baff3f7fb1f8d60c14f3457bc172dd15c8fb Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 11 Mar 2025 23:49:39 +0800 Subject: [PATCH 3/4] fix --- .../common/cache/NereidsSqlCacheManager.java | 20 +++++++++++++++++++ .../LogicalPlanBuilderForEncryption.java | 8 +++++--- .../trees/plans/commands/ExecuteCommand.java | 12 ++++++++--- .../org/apache/doris/qe/ConnectProcessor.java | 11 ++++++++++ .../doris/regression/action/TestAction.groovy | 2 +- 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java index ed9a9f0c00cd9f..e8af373cd7626a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java @@ -32,6 +32,7 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.metric.MetricRepo; +import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.mysql.privilege.DataMaskPolicy; import org.apache.doris.mysql.privilege.RowFilterPolicy; import org.apache.doris.nereids.CascadesContext; @@ -130,6 +131,13 @@ private static Cache buildSqlCaches(int sqlCacheNum, lo * tryAddFeCache */ public void tryAddFeSqlCache(ConnectContext connectContext, String sql) { + switch (connectContext.getCommand()) { + case COM_STMT_EXECUTE: + case COM_STMT_PREPARE: + return; + default: {} + } + Optional sqlCacheContextOpt = connectContext.getStatementContext().getSqlCacheContext(); if (!sqlCacheContextOpt.isPresent()) { return; @@ -149,6 +157,12 @@ public void tryAddFeSqlCache(ConnectContext connectContext, String sql) { * tryAddBeCache */ public void tryAddBeCache(ConnectContext connectContext, String sql, CacheAnalyzer analyzer) { + switch (connectContext.getCommand()) { + case COM_STMT_EXECUTE: + case COM_STMT_PREPARE: + return; + default: {} + } Optional sqlCacheContextOpt = connectContext.getStatementContext().getSqlCacheContext(); if (!sqlCacheContextOpt.isPresent()) { return; @@ -180,6 +194,12 @@ public void tryAddBeCache(ConnectContext connectContext, String sql, CacheAnalyz * tryParseSql */ public Optional tryParseSql(ConnectContext connectContext, String sql) { + switch (connectContext.getCommand()) { + case COM_STMT_EXECUTE: + case COM_STMT_PREPARE: + return Optional.empty(); + default: {} + } String key = generateCacheKey(connectContext, normalizeSql(sql.trim())); SqlCacheContext sqlCacheContext = sqlCaches.getIfPresent(key); if (sqlCacheContext == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java index 809ab277a18b11..1f00fa048d36bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilderForEncryption.java @@ -107,9 +107,11 @@ public LogicalPlan visitCreateTable(DorisParser.CreateTableContext ctx) { if (ctx.propertyClause() != null) { List propertyClauseContexts = ctx.propertyClause(); for (DorisParser.PropertyClauseContext propertyClauseContext : propertyClauseContexts) { - encryptProperty(visitPropertyClause(propertyClauseContext), - propertyClauseContext.fileProperties.start.getStartIndex(), - propertyClauseContext.fileProperties.stop.getStopIndex()); + if (propertyClauseContext != null) { + encryptProperty(visitPropertyClause(propertyClauseContext), + propertyClauseContext.fileProperties.start.getStartIndex(), + propertyClauseContext.fileProperties.stop.getStopIndex()); + } } } return super.visitCreateTable(ctx); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java index c4031c0f9e59b8..c67ab1597cc1dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommand.java @@ -25,6 +25,8 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.insert.OlapGroupCommitInsertExecutor; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.planner.GroupCommitPlanner; import org.apache.doris.qe.ConnectContext; @@ -68,9 +70,13 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { throw new AnalysisException( "prepare statement " + stmtName + " not found, maybe expired"); } - PrepareCommand prepareCommand = (PrepareCommand) preparedStmtCtx.command; - LogicalPlanAdapter planAdapter = new LogicalPlanAdapter(prepareCommand.getLogicalPlan(), executor.getContext() - .getStatementContext()); + PrepareCommand prepareCommand = preparedStmtCtx.command; + LogicalPlan logicalPlan = prepareCommand.getLogicalPlan(); + if (logicalPlan instanceof LogicalSqlCache) { + throw new AnalysisException("Unsupported sql cache for server prepared statement"); + } + LogicalPlanAdapter planAdapter = new LogicalPlanAdapter( + logicalPlan, executor.getContext().getStatementContext()); executor.setParsedStmt(planAdapter); // If it's not a short circuit query or schema version is different(indicates schema changed) or // has nondeterministic functions in statement, then need to do reanalyze and plan diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index fa39a7f6166724..a347231561b027 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -393,6 +393,17 @@ public void executeQuery(String originStmt) throws Exception { private List parseFromSqlCache(String originStmt) { StatementContext statementContext = new StatementContext(ctx, new OriginStatement(originStmt, 0)); ctx.setStatementContext(statementContext); + + // the mysql protocol has different between COM_QUERY and COM_STMT_EXECUTE, + // the sql cache use the result of COM_QUERY, so we can not provide the + // result of sql cache for COM_STMT_EXECUTE/COM_STMT_PREPARE + switch (ctx.getCommand()) { + case COM_STMT_EXECUTE: + case COM_STMT_PREPARE: + return null; + default: {} + } + try { Optional> explainPlan = NereidsParser.tryParseExplainPlan(originStmt); String cacheSqlKey = originStmt; diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy index 3febe6bbad57cf..ebee498da98c85 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/TestAction.groovy @@ -68,7 +68,7 @@ class TestAction implements SuiteAction { } else { if (exception != null || result.exception != null) { def msg = result.exception?.toString() - log.info("Exception: ${msg}") + log.error("Exception: ${msg}", exception != null ? exception : result.exception) Assert.assertTrue("Expect exception msg contains '${exception}', but meet '${msg}'", msg != null && exception != null && msg.contains(exception)) } From 9965643ef2d1d5c4a0981977b482a22b50e88e4e Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 12 Mar 2025 10:57:24 +0800 Subject: [PATCH 4/4] fix --- .../apache/doris/common/cache/NereidsSqlCacheManager.java | 7 +++---- .../main/java/org/apache/doris/qe/ConnectProcessor.java | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java index e8af373cd7626a..5aecf5ff4bbaad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java @@ -32,7 +32,6 @@ import org.apache.doris.common.util.DebugUtil; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.metric.MetricRepo; -import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.mysql.privilege.DataMaskPolicy; import org.apache.doris.mysql.privilege.RowFilterPolicy; import org.apache.doris.nereids.CascadesContext; @@ -135,7 +134,7 @@ public void tryAddFeSqlCache(ConnectContext connectContext, String sql) { case COM_STMT_EXECUTE: case COM_STMT_PREPARE: return; - default: {} + default: { } } Optional sqlCacheContextOpt = connectContext.getStatementContext().getSqlCacheContext(); @@ -161,7 +160,7 @@ public void tryAddBeCache(ConnectContext connectContext, String sql, CacheAnalyz case COM_STMT_EXECUTE: case COM_STMT_PREPARE: return; - default: {} + default: { } } Optional sqlCacheContextOpt = connectContext.getStatementContext().getSqlCacheContext(); if (!sqlCacheContextOpt.isPresent()) { @@ -198,7 +197,7 @@ public Optional tryParseSql(ConnectContext connectContext, Stri case COM_STMT_EXECUTE: case COM_STMT_PREPARE: return Optional.empty(); - default: {} + default: { } } String key = generateCacheKey(connectContext, normalizeSql(sql.trim())); SqlCacheContext sqlCacheContext = sqlCaches.getIfPresent(key); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index a347231561b027..eb5076943a212a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -401,7 +401,7 @@ private List parseFromSqlCache(String originStmt) { case COM_STMT_EXECUTE: case COM_STMT_PREPARE: return null; - default: {} + default: { } } try {