From 1320c97680ce98fd7742d2075929e24801256601 Mon Sep 17 00:00:00 2001
From: deardeng <565620795@qq.com>
Date: Mon, 6 May 2024 19:43:35 +0800
Subject: [PATCH 1/2] [fix](log) Compatible logging strategy for cloud and
 non-cloud

---
 .../java/org/apache/doris/common/Config.java  |   5 +
 .../org/apache/doris/common/Log4jConfig.java  | 185 +++++++++++-------
 2 files changed, 114 insertions(+), 76 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 6a8f2e845dd0f3..5fea7a8472f413 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2612,6 +2612,11 @@ public class Config extends ConfigBase {
     //==========================================================================
     //                    begin of cloud config
     //==========================================================================
+    @ConfField(description = {"是否启用FE 日志文件按照大小删除策略，当日志大小超过指定大小，删除相关的log。默认关闭，为按照时间策略删除",
+        "Whether to enable the FE log file deletion policy based on size, "
+            + "where logs exceeding the specified size are deleted. "
+            + "It is disabled by default and follows a time-based deletion policy."})
+    public static boolean log_by_size = false;
 
     @ConfField public static int info_sys_accumulated_file_size = 4;
     @ConfField public static int warn_sys_accumulated_file_size = 2;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
index 320cdd6114bcef..9ab546cdf360d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
@@ -39,81 +39,114 @@
 public class Log4jConfig extends XmlConfiguration {
     private static final long serialVersionUID = 1L;
 
-    // CHECKSTYLE OFF
-    private static String xmlConfTemplate = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
-            + "\n<!-- Auto Generated. DO NOT MODIFY IT! -->\n"
-            + "<Configuration status=\"info\" packages=\"org.apache.doris.common\">\n"
-            + "  <Appenders>\n"
-            + "    <Console name=\"Console\" target=\"SYSTEM_OUT\">"
-            + "      <PatternLayout charset=\"UTF-8\">\n"
-            + "        <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n"
-            + "      </PatternLayout>\n"
-            + "    </Console>"
-            + "    <RollingFile name=\"Sys\" fileName=\"${sys_log_dir}/fe.log\" filePattern=\"${sys_log_dir}/fe.log.${sys_file_pattern}-%i${sys_file_postfix}\" immediateFlush=\"${immediate_flush_flag}\">\n"
-            + "      <PatternLayout charset=\"UTF-8\">\n"
-            + "        <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n"
-            + "      </PatternLayout>\n"
-            + "      <Policies>\n"
-            + "        <TimeBasedTriggeringPolicy/>\n"
-            + "        <SizeBasedTriggeringPolicy size=\"${sys_roll_maxsize}MB\"/>\n"
-            + "      </Policies>\n"
-            + "      <DefaultRolloverStrategy max=\"${sys_roll_num}\" fileIndex=\"max\">\n"
-            + "        <Delete basePath=\"${sys_log_dir}/\" maxDepth=\"1\">\n"
-            + "          <IfFileName glob=\"fe.log.*\" />\n"
-            + "          <IfAny>\n"
-            + "             <IfAccumulatedFileSize exceeds=\"${info_sys_accumulated_file_size}GB\"/>\n"
-            + "           </IfAny>\n"
-            + "        </Delete>\n"
-            + "      </DefaultRolloverStrategy>\n"
-            + "    </RollingFile>\n"
-            + "    <RollingFile name=\"SysWF\" fileName=\"${sys_log_dir}/fe.warn.log\" filePattern=\"${sys_log_dir}/fe.warn.log.${sys_file_pattern}-%i${sys_file_postfix}\" immediateFlush=\"${immediate_flush_flag}\">\n"
-            + "      <PatternLayout charset=\"UTF-8\">\n"
-            + "        <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n"
-            + "      </PatternLayout>\n"
-            + "      <Policies>\n"
-            + "        <TimeBasedTriggeringPolicy/>\n"
-            + "        <SizeBasedTriggeringPolicy size=\"${sys_roll_maxsize}MB\"/>\n"
-            + "      </Policies>\n"
-            + "      <DefaultRolloverStrategy max=\"${sys_roll_num}\" fileIndex=\"max\">\n"
-            + "        <Delete basePath=\"${sys_log_dir}/\" maxDepth=\"1\">\n"
-            + "          <IfFileName glob=\"fe.warn.log.*\" />\n"
-            + "          <IfAny>\n"
-            + "             <IfAccumulatedFileSize exceeds=\"${warn_sys_accumulated_file_size}GB\"/>\n"
-            + "          </IfAny>\n"
-            + "        </Delete>\n"
-            + "      </DefaultRolloverStrategy>\n"
-            + "    </RollingFile>\n"
-            + "    <RollingFile name=\"Auditfile\" fileName=\"${audit_log_dir}/fe.audit.log\" filePattern=\"${audit_log_dir}/fe.audit.log.${audit_file_pattern}-%i${audit_file_postfix}\">\n"
-            + "      <PatternLayout charset=\"UTF-8\">\n"
-            + "        <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n</Pattern>\n"
-            + "      </PatternLayout>\n"
-            + "      <Policies>\n"
-            + "        <TimeBasedTriggeringPolicy/>\n"
-            + "        <SizeBasedTriggeringPolicy size=\"${audit_roll_maxsize}MB\"/>\n"
-            + "      </Policies>\n"
-            + "      <DefaultRolloverStrategy max=\"${audit_roll_num}\" fileIndex=\"max\">\n"
-            + "        <Delete basePath=\"${audit_log_dir}/\" maxDepth=\"1\">\n"
-            + "          <IfFileName glob=\"fe.audit.log.*\" />\n"
-            + "          <IfAny>\n"
-            + "             <IfAccumulatedFileSize exceeds=\"${audit_sys_accumulated_file_size}GB\"/>\n"
-            + "          </IfAny>\n"
-            + "        </Delete>\n"
-            + "      </DefaultRolloverStrategy>\n"
-            + "    </RollingFile>\n"
-            + "  </Appenders>\n"
-            + "  <Loggers>\n"
-            + "    <Root level=\"${sys_log_level}\" includeLocation=\"${include_location_flag}\">\n"
-            + "      <AppenderRef ref=\"Sys\"/>\n"
-            + "      <AppenderRef ref=\"SysWF\" level=\"WARN\"/>\n"
-            + "      <!--REPLACED BY Console Logger-->\n"
-            + "    </Root>\n"
-            + "    <Logger name=\"audit\" level=\"ERROR\" additivity=\"false\">\n"
-            + "      <AppenderRef ref=\"Auditfile\"/>\n"
-            + "    </Logger>\n"
-            + "    <!--REPLACED BY AUDIT AND VERBOSE MODULE NAMES-->\n"
-            + "  </Loggers>\n"
-            + "</Configuration>";
-    // CHECKSTYLE ON
+    private static StringBuilder xmlConfTemplateBuilder = new StringBuilder();
+
+    static {
+        // CHECKSTYLE OFF
+        xmlConfTemplateBuilder.append("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
+            .append("\n<!-- Auto Generated. DO NOT MODIFY IT! -->\n")
+            .append("<Configuration status=\"info\" packages=\"org.apache.doris.common\">\n")
+            .append("  <Appenders>\n")
+            .append("    <Console name=\"Console\" target=\"SYSTEM_OUT\">")
+            .append("      <PatternLayout charset=\"UTF-8\">\n")
+            .append("        <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n")
+            .append("      </PatternLayout>\n")
+            .append("    </Console>")
+            .append("    <RollingFile name=\"Sys\" fileName=\"${sys_log_dir}/fe.log\" filePattern=\"${sys_log_dir}/fe.log.${sys_file_pattern}-%i${sys_file_postfix}\" immediateFlush=\"${immediate_flush_flag}\">\n")
+            .append("      <PatternLayout charset=\"UTF-8\">\n")
+            .append("        <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n")
+            .append("      </PatternLayout>\n")
+            .append("      <Policies>\n")
+            .append("        <TimeBasedTriggeringPolicy/>\n")
+            .append("        <SizeBasedTriggeringPolicy size=\"${sys_roll_maxsize}MB\"/>\n")
+            .append("      </Policies>\n")
+            .append("      <DefaultRolloverStrategy max=\"${sys_roll_num}\" fileIndex=\"max\">\n")
+            .append("        <Delete basePath=\"${sys_log_dir}/\" maxDepth=\"1\">\n")
+            .append("          <IfFileName glob=\"fe.log.*\" />\n");
+
+        if (Config.log_by_size) {
+            xmlConfTemplateBuilder
+                .append("          <IfAny>\n")
+                .append("             <IfAccumulatedFileSize exceeds=\"${info_sys_accumulated_file_size}GB\"/>\n")
+                .append("           </IfAny>\n");
+
+        } else {
+            xmlConfTemplateBuilder
+                .append("          <IfLastModified age=\"${sys_log_delete_age}\" />\n");
+        }
+
+        xmlConfTemplateBuilder
+            .append("        </Delete>\n")
+            .append("      </DefaultRolloverStrategy>\n")
+            .append("    </RollingFile>\n")
+            .append("    <RollingFile name=\"SysWF\" fileName=\"${sys_log_dir}/fe.warn.log\" filePattern=\"${sys_log_dir}/fe.warn.log.${sys_file_pattern}-%i${sys_file_postfix}\" immediateFlush=\"${immediate_flush_flag}\">\n")
+            .append("      <PatternLayout charset=\"UTF-8\">\n")
+            .append("        <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n")
+            .append("      </PatternLayout>\n")
+            .append("      <Policies>\n")
+            .append("        <TimeBasedTriggeringPolicy/>\n")
+            .append("        <SizeBasedTriggeringPolicy size=\"${sys_roll_maxsize}MB\"/>\n")
+            .append("      </Policies>\n")
+            .append("      <DefaultRolloverStrategy max=\"${sys_roll_num}\" fileIndex=\"max\">\n")
+            .append("        <Delete basePath=\"${sys_log_dir}/\" maxDepth=\"1\">\n")
+            .append("          <IfFileName glob=\"fe.warn.log.*\" />\n");
+
+        if (Config.log_by_size) {
+            xmlConfTemplateBuilder
+                .append("          <IfAny>\n")
+                .append("             <IfAccumulatedFileSize exceeds=\"${warn_sys_accumulated_file_size}GB\"/>\n")
+                .append("          </IfAny>\n");
+        } else {
+            xmlConfTemplateBuilder
+                .append("          <IfLastModified age=\"${sys_log_delete_age}\" />\n");
+        }
+
+        xmlConfTemplateBuilder
+            .append("        </Delete>\n")
+            .append("      </DefaultRolloverStrategy>\n")
+            .append("    </RollingFile>\n")
+            .append("    <RollingFile name=\"Auditfile\" fileName=\"${audit_log_dir}/fe.audit.log\" filePattern=\"${audit_log_dir}/fe.audit.log.${audit_file_pattern}-%i${audit_file_postfix}\">\n")
+            .append("      <PatternLayout charset=\"UTF-8\">\n")
+            .append("        <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n</Pattern>\n")
+            .append("      </PatternLayout>\n")
+            .append("      <Policies>\n")
+            .append("        <TimeBasedTriggeringPolicy/>\n")
+            .append("        <SizeBasedTriggeringPolicy size=\"${audit_roll_maxsize}MB\"/>\n")
+            .append("      </Policies>\n")
+            .append("      <DefaultRolloverStrategy max=\"${audit_roll_num}\" fileIndex=\"max\">\n")
+            .append("        <Delete basePath=\"${audit_log_dir}/\" maxDepth=\"1\">\n")
+            .append("          <IfFileName glob=\"fe.audit.log.*\" />\n");
+
+        if (Config.log_by_size) {
+            xmlConfTemplateBuilder
+                .append("          <IfAny>\n")
+                .append("             <IfAccumulatedFileSize exceeds=\"${audit_sys_accumulated_file_size}GB\"/>\n")
+                .append("          </IfAny>\n");
+
+        } else {
+            xmlConfTemplateBuilder
+                .append("          <IfLastModified age=\"${audit_log_delete_age}\" />\n");
+        }
+
+        xmlConfTemplateBuilder
+            .append("        </Delete>\n")
+            .append("      </DefaultRolloverStrategy>\n")
+            .append("    </RollingFile>\n")
+            .append("  </Appenders>\n")
+            .append("  <Loggers>\n")
+            .append("    <Root level=\"${sys_log_level}\" includeLocation=\"${include_location_flag}\">\n")
+            .append("      <AppenderRef ref=\"Sys\"/>\n")
+            .append("      <AppenderRef ref=\"SysWF\" level=\"WARN\"/>\n")
+            .append("      <!--REPLACED BY Console Logger-->\n")
+            .append("    </Root>\n")
+            .append("    <Logger name=\"audit\" level=\"ERROR\" additivity=\"false\">\n")
+            .append("      <AppenderRef ref=\"Auditfile\"/>\n")
+            .append("    </Logger>\n")
+            .append("    <!--REPLACED BY AUDIT AND VERBOSE MODULE NAMES-->\n")
+            .append("  </Loggers>\n")
+            .append("</Configuration>");
+        // CHECKSTYLE ON
+    }
 
     private static StrSubstitutor strSub;
     private static String sysLogLevel;
@@ -134,7 +167,7 @@ public class Log4jConfig extends XmlConfiguration {
     public static boolean foreground = false;
 
     private static void reconfig() throws IOException {
-        String newXmlConfTemplate = xmlConfTemplate;
+        String newXmlConfTemplate = xmlConfTemplateBuilder.toString();
 
         // sys log config
         // ATTN, sys_log_dir is deprecated, use LOG_DIR instead

From a3aeb240373ace15d081d829a144fb206dc81dc2 Mon Sep 17 00:00:00 2001
From: deardeng <565620795@qq.com>
Date: Tue, 7 May 2024 11:00:51 +0800
Subject: [PATCH 2/2] fix review

---
 .../java/org/apache/doris/common/Config.java  |  7 +--
 .../org/apache/doris/common/Log4jConfig.java  | 47 +++++++------------
 2 files changed, 21 insertions(+), 33 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 5fea7a8472f413..4dd98b28da11e5 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2612,11 +2612,12 @@ public class Config extends ConfigBase {
     //==========================================================================
     //                    begin of cloud config
     //==========================================================================
-    @ConfField(description = {"是否启用FE 日志文件按照大小删除策略，当日志大小超过指定大小，删除相关的log。默认关闭，为按照时间策略删除",
+    @ConfField(description = {"是否启用FE 日志文件按照大小删除策略，当日志大小超过指定大小，删除相关的log。默认为按照时间策略删除",
         "Whether to enable the FE log file deletion policy based on size, "
             + "where logs exceeding the specified size are deleted. "
-            + "It is disabled by default and follows a time-based deletion policy."})
-    public static boolean log_by_size = false;
+            + "It is disabled by default and follows a time-based deletion policy."},
+            options = {"age", "size"})
+    public static String log_rollover_strategy = "age";
 
     @ConfField public static int info_sys_accumulated_file_size = 4;
     @ConfField public static int warn_sys_accumulated_file_size = 2;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
index 9ab546cdf360d4..8c840a876bb69a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
@@ -41,6 +41,19 @@ public class Log4jConfig extends XmlConfiguration {
 
     private static StringBuilder xmlConfTemplateBuilder = new StringBuilder();
 
+    private static void getXmlConfByStrategy(final String size, final String age) {
+        if (Config.log_rollover_strategy.equalsIgnoreCase("size")) {
+            xmlConfTemplateBuilder
+                    .append("          <IfAny>\n")
+                    .append("             <IfAccumulatedFileSize exceeds=\"${").append(size).append("}GB\"/>\n")
+                    .append("           </IfAny>\n");
+        } else {
+            // default age
+            xmlConfTemplateBuilder
+                    .append("          <IfLastModified age=\"${").append(age).append("}\" />\n");
+        }
+    }
+
     static {
         // CHECKSTYLE OFF
         xmlConfTemplateBuilder.append("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
@@ -51,7 +64,7 @@ public class Log4jConfig extends XmlConfiguration {
             .append("      <PatternLayout charset=\"UTF-8\">\n")
             .append("        <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n")
             .append("      </PatternLayout>\n")
-            .append("    </Console>")
+            .append("    </Console>\n")
             .append("    <RollingFile name=\"Sys\" fileName=\"${sys_log_dir}/fe.log\" filePattern=\"${sys_log_dir}/fe.log.${sys_file_pattern}-%i${sys_file_postfix}\" immediateFlush=\"${immediate_flush_flag}\">\n")
             .append("      <PatternLayout charset=\"UTF-8\">\n")
             .append("        <Pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)<!--REPLACED BY LOG FORMAT-->%m%n</Pattern>\n")
@@ -64,16 +77,7 @@ public class Log4jConfig extends XmlConfiguration {
             .append("        <Delete basePath=\"${sys_log_dir}/\" maxDepth=\"1\">\n")
             .append("          <IfFileName glob=\"fe.log.*\" />\n");
 
-        if (Config.log_by_size) {
-            xmlConfTemplateBuilder
-                .append("          <IfAny>\n")
-                .append("             <IfAccumulatedFileSize exceeds=\"${info_sys_accumulated_file_size}GB\"/>\n")
-                .append("           </IfAny>\n");
-
-        } else {
-            xmlConfTemplateBuilder
-                .append("          <IfLastModified age=\"${sys_log_delete_age}\" />\n");
-        }
+        getXmlConfByStrategy("info_sys_accumulated_file_size", "sys_log_delete_age");
 
         xmlConfTemplateBuilder
             .append("        </Delete>\n")
@@ -91,15 +95,7 @@ public class Log4jConfig extends XmlConfiguration {
             .append("        <Delete basePath=\"${sys_log_dir}/\" maxDepth=\"1\">\n")
             .append("          <IfFileName glob=\"fe.warn.log.*\" />\n");
 
-        if (Config.log_by_size) {
-            xmlConfTemplateBuilder
-                .append("          <IfAny>\n")
-                .append("             <IfAccumulatedFileSize exceeds=\"${warn_sys_accumulated_file_size}GB\"/>\n")
-                .append("          </IfAny>\n");
-        } else {
-            xmlConfTemplateBuilder
-                .append("          <IfLastModified age=\"${sys_log_delete_age}\" />\n");
-        }
+        getXmlConfByStrategy("warn_sys_accumulated_file_size", "sys_log_delete_age");
 
         xmlConfTemplateBuilder
             .append("        </Delete>\n")
@@ -117,16 +113,7 @@ public class Log4jConfig extends XmlConfiguration {
             .append("        <Delete basePath=\"${audit_log_dir}/\" maxDepth=\"1\">\n")
             .append("          <IfFileName glob=\"fe.audit.log.*\" />\n");
 
-        if (Config.log_by_size) {
-            xmlConfTemplateBuilder
-                .append("          <IfAny>\n")
-                .append("             <IfAccumulatedFileSize exceeds=\"${audit_sys_accumulated_file_size}GB\"/>\n")
-                .append("          </IfAny>\n");
-
-        } else {
-            xmlConfTemplateBuilder
-                .append("          <IfLastModified age=\"${audit_log_delete_age}\" />\n");
-        }
+        getXmlConfByStrategy("audit_sys_accumulated_file_size", "audit_log_delete_age");
 
         xmlConfTemplateBuilder
             .append("        </Delete>\n")