From 5474a47492c314a589b84e59fe7db6384ab2ba10 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Fri, 26 Jul 2024 17:50:55 +0800 Subject: [PATCH 1/5] 1 --- .../org/apache/doris/qe/MysqlConnectProcessor.java | 14 ++++++++++---- .../java/org/apache/doris/qe/SessionVariable.java | 10 ++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index b772da388db249..38db7813ce461d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -136,11 +136,17 @@ private void handleExecute(PrepareStmt prepareStmt, long stmtId) { executor = new StmtExecutor(ctx, executeStmt); ctx.setExecutor(executor); executor.execute(); - PrepareStmtContext preparedStmtContext = ConnectContext.get().getPreparedStmt(String.valueOf(stmtId)); - if (preparedStmtContext != null) { - stmtStr = executeStmt.toSql(); + //For the `insert into` statements during group commit load via JDBC. + //Printing audit logs can severely impact performance. + //Therefore, we have introduced a session variable to control whether to print audit logs. + //It is recommended to turn off audit logs only during group commit load via JDBC. + if (ctx.getSessionVariable().isEnableAuditLog()) { + PrepareStmtContext preparedStmtContext = ConnectContext.get().getPreparedStmt(String.valueOf(stmtId)); + if (preparedStmtContext != null) { + stmtStr = executeStmt.toSql(); + } } - } catch (Throwable e) { + } catch (Throwable e) { // Catch all throwable. // If reach here, maybe doris bug. LOG.warn("Process one query failed because unknown reason: ", e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 46fb386fc30edc..5a4b9a07b45555 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -473,8 +473,11 @@ public class SessionVariable implements Serializable, Writable { public static final String EXTERNAL_TABLE_ANALYZE_PART_NUM = "external_table_analyze_part_num"; public static final String ENABLE_STRONG_CONSISTENCY = "enable_strong_consistency_read"; + public static final String GROUP_COMMIT = "group_commit"; + public static final String ENABLE_AUDIT_LOG = "enable_audit_log"; + public static final String PARALLEL_SYNC_ANALYZE_TASK_NUM = "parallel_sync_analyze_task_num"; public static final String TRUNCATE_CHAR_OR_VARCHAR_COLUMNS = "truncate_char_or_varchar_columns"; @@ -1679,6 +1682,9 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { @VariableMgr.VarAttr(name = GROUP_COMMIT, needForward = true) public String groupCommit = "off_mode"; + @VariableMgr.VarAttr(name = ENABLE_AUDIT_LOG, needForward = true) + public boolean enableAuditLog = true; + @VariableMgr.VarAttr(name = INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD, description = {"在match_all中求取多个倒排索引的交集时,如果最大的倒排索引中的总数是最小倒排索引中的总数的整数倍," + "则使用跳表来优化交集操作。", @@ -4132,6 +4138,10 @@ public String getGroupCommit() { return groupCommit; } + public boolean isEnableAuditLog() { + return enableAuditLog; + } + public boolean isEnableMaterializedViewRewrite() { return enableMaterializedViewRewrite; } From 96e0073e14fe9e67b0e7f171da74ba5d1be47b51 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Mon, 29 Jul 2024 13:09:28 +0800 Subject: [PATCH 2/5] 2 --- .../org/apache/doris/qe/MysqlConnectProcessor.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index 38db7813ce461d..359e441cd79934 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -153,7 +153,9 @@ private void handleExecute(PrepareStmt prepareStmt, long stmtId) { ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getClass().getSimpleName() + ", msg: " + e.getMessage()); } - auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); + if (ctx.getSessionVariable().isEnableAuditLog()) { + auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); + } } private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedStatementContext prepCtx) { @@ -205,15 +207,19 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS executor = new StmtExecutor(ctx, stmt); ctx.setExecutor(executor); executor.execute(); - stmtStr = executeStmt.toSql(); - } catch (Throwable e) { + if (ctx.getSessionVariable().isEnableAuditLog()) { + stmtStr = executeStmt.toSql(); + } + } catch (Throwable e) { // Catch all throwable. // If reach here, maybe doris bug. LOG.warn("Process one query failed because unknown reason: ", e); ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getClass().getSimpleName() + ", msg: " + e.getMessage()); } - auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); + if (ctx.getSessionVariable().isEnableAuditLog()) { + auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); + } } // process COM_EXECUTE, parse binary row data From 404c727eb24bf98c1f2a040b576a8778b21dd134 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Mon, 29 Jul 2024 13:12:35 +0800 Subject: [PATCH 3/5] 3 --- .../java/org/apache/doris/qe/MysqlConnectProcessor.java | 8 ++++---- .../main/java/org/apache/doris/qe/SessionVariable.java | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index 359e441cd79934..adf272482bd698 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -140,7 +140,7 @@ private void handleExecute(PrepareStmt prepareStmt, long stmtId) { //Printing audit logs can severely impact performance. //Therefore, we have introduced a session variable to control whether to print audit logs. //It is recommended to turn off audit logs only during group commit load via JDBC. - if (ctx.getSessionVariable().isEnableAuditLog()) { + if (ctx.getSessionVariable().isEnableExecStmtAuditLog()) { PrepareStmtContext preparedStmtContext = ConnectContext.get().getPreparedStmt(String.valueOf(stmtId)); if (preparedStmtContext != null) { stmtStr = executeStmt.toSql(); @@ -153,7 +153,7 @@ private void handleExecute(PrepareStmt prepareStmt, long stmtId) { ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getClass().getSimpleName() + ", msg: " + e.getMessage()); } - if (ctx.getSessionVariable().isEnableAuditLog()) { + if (ctx.getSessionVariable().isEnableExecStmtAuditLog()) { auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); } } @@ -207,7 +207,7 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS executor = new StmtExecutor(ctx, stmt); ctx.setExecutor(executor); executor.execute(); - if (ctx.getSessionVariable().isEnableAuditLog()) { + if (ctx.getSessionVariable().isEnableExecStmtAuditLog()) { stmtStr = executeStmt.toSql(); } } catch (Throwable e) { @@ -217,7 +217,7 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getClass().getSimpleName() + ", msg: " + e.getMessage()); } - if (ctx.getSessionVariable().isEnableAuditLog()) { + if (ctx.getSessionVariable().isEnableExecStmtAuditLog()) { auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 5a4b9a07b45555..b1fe8fdb1f59d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1683,7 +1683,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { public String groupCommit = "off_mode"; @VariableMgr.VarAttr(name = ENABLE_AUDIT_LOG, needForward = true) - public boolean enableAuditLog = true; + public boolean enableExecStmtAuditLog = true; @VariableMgr.VarAttr(name = INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD, description = {"在match_all中求取多个倒排索引的交集时,如果最大的倒排索引中的总数是最小倒排索引中的总数的整数倍," @@ -4138,8 +4138,8 @@ public String getGroupCommit() { return groupCommit; } - public boolean isEnableAuditLog() { - return enableAuditLog; + public boolean isEnableExecStmtAuditLog() { + return enableExecStmtAuditLog; } public boolean isEnableMaterializedViewRewrite() { From c9ea6aaf9dca357b0c66d5b4c0d0367ebeedbdd2 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Mon, 29 Jul 2024 14:15:05 +0800 Subject: [PATCH 4/5] 4 --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index b1fe8fdb1f59d6..f98e4c0584fb01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -476,7 +476,7 @@ public class SessionVariable implements Serializable, Writable { public static final String GROUP_COMMIT = "group_commit"; - public static final String ENABLE_AUDIT_LOG = "enable_audit_log"; + public static final String ENABLE_EXEC_STMT_AUDIT_LOG = "enable_exec_stmt_audit_log"; public static final String PARALLEL_SYNC_ANALYZE_TASK_NUM = "parallel_sync_analyze_task_num"; @@ -1682,7 +1682,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { @VariableMgr.VarAttr(name = GROUP_COMMIT, needForward = true) public String groupCommit = "off_mode"; - @VariableMgr.VarAttr(name = ENABLE_AUDIT_LOG, needForward = true) + @VariableMgr.VarAttr(name = ENABLE_EXEC_STMT_AUDIT_LOG, needForward = true) public boolean enableExecStmtAuditLog = true; @VariableMgr.VarAttr(name = INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD, From 736ad17b7677a5057acd589007ed0a244611df09 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Mon, 29 Jul 2024 17:07:54 +0800 Subject: [PATCH 5/5] 5 --- .../org/apache/doris/qe/MysqlConnectProcessor.java | 8 ++++---- .../main/java/org/apache/doris/qe/SessionVariable.java | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index adf272482bd698..165767c8611859 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -140,7 +140,7 @@ private void handleExecute(PrepareStmt prepareStmt, long stmtId) { //Printing audit logs can severely impact performance. //Therefore, we have introduced a session variable to control whether to print audit logs. //It is recommended to turn off audit logs only during group commit load via JDBC. - if (ctx.getSessionVariable().isEnableExecStmtAuditLog()) { + if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) { PrepareStmtContext preparedStmtContext = ConnectContext.get().getPreparedStmt(String.valueOf(stmtId)); if (preparedStmtContext != null) { stmtStr = executeStmt.toSql(); @@ -153,7 +153,7 @@ private void handleExecute(PrepareStmt prepareStmt, long stmtId) { ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getClass().getSimpleName() + ", msg: " + e.getMessage()); } - if (ctx.getSessionVariable().isEnableExecStmtAuditLog()) { + if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) { auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); } } @@ -207,7 +207,7 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS executor = new StmtExecutor(ctx, stmt); ctx.setExecutor(executor); executor.execute(); - if (ctx.getSessionVariable().isEnableExecStmtAuditLog()) { + if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) { stmtStr = executeStmt.toSql(); } } catch (Throwable e) { @@ -217,7 +217,7 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getClass().getSimpleName() + ", msg: " + e.getMessage()); } - if (ctx.getSessionVariable().isEnableExecStmtAuditLog()) { + if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) { auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index f98e4c0584fb01..692abbcd82fff4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -476,7 +476,7 @@ public class SessionVariable implements Serializable, Writable { public static final String GROUP_COMMIT = "group_commit"; - public static final String ENABLE_EXEC_STMT_AUDIT_LOG = "enable_exec_stmt_audit_log"; + public static final String ENABLE_PREPARED_STMT_AUDIT_LOG = "enable_prepared_stmt_audit_log"; public static final String PARALLEL_SYNC_ANALYZE_TASK_NUM = "parallel_sync_analyze_task_num"; @@ -1682,8 +1682,8 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { @VariableMgr.VarAttr(name = GROUP_COMMIT, needForward = true) public String groupCommit = "off_mode"; - @VariableMgr.VarAttr(name = ENABLE_EXEC_STMT_AUDIT_LOG, needForward = true) - public boolean enableExecStmtAuditLog = true; + @VariableMgr.VarAttr(name = ENABLE_PREPARED_STMT_AUDIT_LOG, needForward = true) + public boolean enablePreparedStmtAuditLog = true; @VariableMgr.VarAttr(name = INVERTED_INDEX_CONJUNCTION_OPT_THRESHOLD, description = {"在match_all中求取多个倒排索引的交集时,如果最大的倒排索引中的总数是最小倒排索引中的总数的整数倍," @@ -4138,8 +4138,8 @@ public String getGroupCommit() { return groupCommit; } - public boolean isEnableExecStmtAuditLog() { - return enableExecStmtAuditLog; + public boolean isEnablePreparedStmtAuditLog() { + return enablePreparedStmtAuditLog; } public boolean isEnableMaterializedViewRewrite() {