[opt](log) refine the FE logger

bin/start_fe.sh

@@ -176,23 +176,34 @@ if [[ ! -d "${LOG_DIR}" ]]; then
     mkdir -p "${LOG_DIR}"
 fi
 
+STDOUT_LOGGER="${LOG_DIR}/fe.out"
+log() {
+    # same datetime format as in fe.log: 2024-06-03 14:54:41,478
+    cur_date=$(date +"%Y-%m-%d %H:%M:%S,$(date +%3N)")
+    if [[ "${RUN_CONSOLE}" -eq 1 ]]; then
+        echo "StdoutLogger ${cur_date} $1"
+    else
+        echo "StdoutLogger ${cur_date} $1" >>"${STDOUT_LOGGER}"
+    fi
+}
+
 # check java version and choose correct JAVA_OPTS
 java_version="$(
     set -e
     jdk_version "${JAVA}"
 )"
 if [[ "${java_version}" -eq 17 ]]; then
     if [[ -z "${JAVA_OPTS_FOR_JDK_17}" ]]; then
-        echo "JAVA_OPTS_FOR_JDK_17 is not set in fe.conf" >>"${LOG_DIR}/fe.out"
+        log "JAVA_OPTS_FOR_JDK_17 is not set in fe.conf"
         exit 1
     fi
     final_java_opt="${JAVA_OPTS_FOR_JDK_17}"
 else
-    echo "ERROR: The jdk_version is ${java_version}, must be 17." >>"${LOG_DIR}/fe.out"
+    log "ERROR: The jdk_version is ${java_version}, must be 17."
     exit 1
 fi
-echo "Using Java version ${java_version}" >>"${LOG_DIR}/fe.out"
-echo "${final_java_opt}" >>"${LOG_DIR}/fe.out"
+log "Using Java version ${java_version}"
+log "${final_java_opt}"
 
 # add libs to CLASSPATH
 DORIS_FE_JAR=
@@ -239,7 +250,8 @@ if [[ -n "${JACOCO_COVERAGE_OPT}" ]]; then
     coverage_opt="${JACOCO_COVERAGE_OPT}"
 fi
 
-date >>"${LOG_DIR}/fe.out"
+CUR_DATE=$(date)
+log "start time: ${CUR_DATE}"
 
 if [[ "${HELPER}" != "" ]]; then
     # change it to '-helper' to be compatible with code in Frontend
@@ -253,12 +265,12 @@ if [[ "${IMAGE_TOOL}" -eq 1 ]]; then
         echo "Internal error, USE IMAGE_TOOL like: ./start_fe.sh --image image_path"
     fi
 elif [[ "${RUN_DAEMON}" -eq 1 ]]; then
-    nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" ${coverage_opt:+${coverage_opt}} org.apache.doris.DorisFE ${HELPER:+${HELPER}} "${METADATA_FAILURE_RECOVERY}" "$@" >>"${LOG_DIR}/fe.out" 2>&1 </dev/null &
+    nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" ${coverage_opt:+${coverage_opt}} org.apache.doris.DorisFE ${HELPER:+${HELPER}} "${METADATA_FAILURE_RECOVERY}" "$@" >>"${STDOUT_LOGGER}" 2>&1 </dev/null &
 elif [[ "${RUN_CONSOLE}" -eq 1 ]]; then
     export DORIS_LOG_TO_STDERR=1
     ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" ${coverage_opt:+${coverage_opt}} org.apache.doris.DorisFE ${HELPER:+${HELPER}} ${OPT_VERSION:+${OPT_VERSION}} "${METADATA_FAILURE_RECOVERY}" "$@" </dev/null
 else
-    ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" ${coverage_opt:+${coverage_opt}} org.apache.doris.DorisFE ${HELPER:+${HELPER}} ${OPT_VERSION:+${OPT_VERSION}} "${METADATA_FAILURE_RECOVERY}" "$@" >>"${LOG_DIR}/fe.out" 2>&1 </dev/null
+    ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" ${coverage_opt:+${coverage_opt}} org.apache.doris.DorisFE ${HELPER:+${HELPER}} ${OPT_VERSION:+${OPT_VERSION}} "${METADATA_FAILURE_RECOVERY}" "$@" >>"${STDOUT_LOGGER}" 2>&1 </dev/null
 fi
 
 if [[ "${OPT_VERSION}" != "" ]]; then

fe/fe-common/pom.xml

@@ -120,6 +120,18 @@ under the License.
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-s3</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-web</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-iostreams</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+        </dependency>
     </dependencies>
     <build>
         <finalName>doris-fe-common</finalName>

fe/fe-common/src/main/java/org/apache/doris/cloud/security/UrlSecurityChecker.java

@@ -18,6 +18,7 @@
 package org.apache.doris.cloud.security;
 
 import org.apache.doris.common.Config;
+import org.apache.doris.common.LogUtils;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -38,8 +39,8 @@ public class UrlSecurityChecker extends SecurityChecker {
             urlSecurityStopCheckMethod = clazz.getMethod("stopSSRFNetHookChecking", String.class);
         } catch (Exception e) {
             LOG.warn("security_checker_class_name:{} exception:", Config.security_checker_class_name, e);
+            LogUtils.stderr("Failed to find com.aliyun.securitysdk.SecurityUtil's method");
             e.printStackTrace();
-            System.out.println("Failed to find com.aliyun.securitysdk.SecurityUtil's method");
         }
     }
 

fe/fe-common/src/main/java/org/apache/doris/common/Config.java

@@ -127,6 +127,13 @@ public class Config extends ConfigBase {
     @ConfField(description = {"是否压缩 FE 的 Audit 日志", "enable compression for FE audit log file"})
     public static boolean audit_log_enable_compress = false;
 
+    @ConfField(description = {"是否使用文件记录日志。当使用 --console 启动 FE 时，全部日志同时写入到标准输出和文件。"
+            + "如果关闭这个选项，不再使用文件记录日志。",
+            "Whether to use file to record log. When starting FE with --console, "
+                    + "all logs will be written to both standard output and file. "
+                    + "Close this option will no longer use file to record log."})
+    public static boolean enable_file_logger = true;
+
     @ConfField(mutable = false, masterOnly = false,
             description = {"是否检查table锁泄漏", "Whether to check table lock leaky"})
     public static boolean check_table_lock_leaky = false;

fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java

@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.common;
+
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.config.Node;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
+import org.apache.logging.log4j.core.config.plugins.PluginFactory;
+import org.apache.logging.log4j.core.layout.AbstractStringLayout;
+import org.apache.logging.log4j.core.layout.PatternLayout;
+import org.apache.logging.log4j.core.util.StringBuilderWriter;
+
+import java.io.PrintWriter;
+import java.nio.charset.Charset;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+
+public class LogUtils {
+
+    public static final String STDOUT_LOG_MARKER = "StdoutLogger ";
+    public static final String STDERR_LOG_MARKER = "StderrLogger ";
+
+    private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS");
+
+    private static String formattedTime() {
+        LocalDateTime dateTime = LocalDateTime.now();
+        return dateTime.format(TIME_FORMATTER);
+    }
+
+    // Developer should use `LogUtils.stdout` or `LogUtils.stderr`
+    // instead of `System.out` and `System.err`.
+    public static void stdout(String message) {
+        System.out.println(STDOUT_LOG_MARKER + formattedTime() + " " + message);
+    }
+
+    public static void stderr(String message) {
+        System.err.println(STDERR_LOG_MARKER + formattedTime() + " " + message);
+    }
+
+    // TODO: this custom layout is not used in the codebase, but it is a good example of how to create a custom layout
+    // 1. Add log4j2.component.properties in fe/conf with content:
+    //      log4j.layoutFactory=org.apache.doris.common.LogUtils$SingleLineExceptionLayout
+    // 2. Change PatternLayout in Log4jConfig.java to SingleLineExceptionLayout
+    @Plugin(name = "SingleLineExceptionLayout", category = Node.CATEGORY,
+            elementType = Layout.ELEMENT_TYPE, printObject = true)
+    public static class SingleLineExceptionLayout extends AbstractStringLayout {
+
+        private final PatternLayout patternLayout;
+
+        protected SingleLineExceptionLayout(PatternLayout patternLayout, Charset charset) {
+            super(charset);
+            this.patternLayout = patternLayout;
+        }
+
+        @Override
+        public String toSerializable(LogEvent event) {
+            StringBuilder result = new StringBuilder(patternLayout.toSerializable(event));
+
+            if (event.getThrown() != null) {
+                StringBuilderWriter sw = new StringBuilderWriter();
+                event.getThrown().printStackTrace(new PrintWriter(sw));
+                String stackTrace = sw.toString().replace("\n", " ").replace("\r", " ");
+                result.append(stackTrace);
+            }
+
+            return result.toString();
+        }
+
+        @PluginFactory
+        public static Layout<String> createLayout(
+                @PluginAttribute(value = "pattern") String pattern,
+                @PluginAttribute(value = "charset", defaultString = "UTF-8") Charset charset) {
+            PatternLayout patternLayout = PatternLayout.newBuilder()
+                    .withPattern(pattern)
+                    .withCharset(charset)
+                    .build();
+            return new SingleLineExceptionLayout(patternLayout, charset);
+        }
+    }
+}

fe/fe-core/pom.xml

@@ -90,11 +90,6 @@ under the License.
             <artifactId>fe-common</artifactId>
             <version>${project.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-web</artifactId>
-            <version>${log4j2.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-devtools</artifactId>

fe/fe-core/src/main/java/org/apache/doris/DorisFE.java

@@ -24,6 +24,7 @@
 import org.apache.doris.common.FeMetaVersion;
 import org.apache.doris.common.LdapConfig;
 import org.apache.doris.common.Log4jConfig;
+import org.apache.doris.common.LogUtils;
 import org.apache.doris.common.ThreadPoolManager;
 import org.apache.doris.common.Version;
 import org.apache.doris.common.util.JdkUtils;
@@ -386,11 +387,11 @@ private static CommandLineOptions parseArgs(String[] args) {
     }
 
     private static void printVersion() {
-        System.out.println("Build version: " + Version.DORIS_BUILD_VERSION);
-        System.out.println("Build time: " + Version.DORIS_BUILD_TIME);
-        System.out.println("Build info: " + Version.DORIS_BUILD_INFO);
-        System.out.println("Build hash: " + Version.DORIS_BUILD_HASH);
-        System.out.println("Java compile version: " + Version.DORIS_JAVA_COMPILE_VERSION);
+        LogUtils.stdout("Build version: " + Version.DORIS_BUILD_VERSION);
+        LogUtils.stdout("Build time: " + Version.DORIS_BUILD_TIME);
+        LogUtils.stdout("Build info: " + Version.DORIS_BUILD_INFO);
+        LogUtils.stdout("Build hash: " + Version.DORIS_BUILD_HASH);
+        LogUtils.stdout("Java compile version: " + Version.DORIS_JAVA_COMPILE_VERSION);
 
         LOG.info("Build version: {}", Version.DORIS_BUILD_VERSION);
         LOG.info("Build time: {}", Version.DORIS_BUILD_TIME);
@@ -413,16 +414,16 @@ private static void checkCommandLineOptions(CommandLineOptions cmdLineOpts) {
         } else if (cmdLineOpts.runImageTool()) {
             File imageFile = new File(cmdLineOpts.getImagePath());
             if (!imageFile.exists()) {
-                System.out.println("image does not exist: " + imageFile.getAbsolutePath()
+                LogUtils.stderr("image does not exist: " + imageFile.getAbsolutePath()
                         + " . Please put an absolute path instead");
                 System.exit(-1);
             } else {
-                System.out.println("Start to load image: ");
+                LogUtils.stdout("Start to load image: ");
                 try {
                     MetaReader.read(imageFile, Env.getCurrentEnv());
-                    System.out.println("Load image success. Image file " + cmdLineOpts.getImagePath() + " is valid");
+                    LogUtils.stdout("Load image success. Image file " + cmdLineOpts.getImagePath() + " is valid");
                 } catch (Exception e) {
-                    System.out.println("Load image failed. Image file " + cmdLineOpts.getImagePath() + " is invalid");
+                    LogUtils.stderr("Load image failed. Image file " + cmdLineOpts.getImagePath() + " is invalid");
                     LOG.warn("", e);
                 } finally {
                     System.exit(0);
@@ -481,9 +482,7 @@ private static void tryLockProcess() {
                 + "same time");
     }
 
-
     private static void releaseFileLockAndCloseFileChannel() {
-
         if (processFileLock != null && processFileLock.isValid()) {
             try {
                 processFileLock.release();
@@ -498,7 +497,6 @@ private static void releaseFileLockAndCloseFileChannel() {
                 LOG.warn("release process lock file failed", ignored);
             }
         }
-
     }
 
     public static void overwriteConfigs() {

fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java

@@ -103,6 +103,7 @@
 import org.apache.doris.common.ErrorReport;
 import org.apache.doris.common.FeConstants;
 import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.LogUtils;
 import org.apache.doris.common.MetaNotFoundException;
 import org.apache.doris.common.NereidsSqlCacheManager;
 import org.apache.doris.common.Pair;
@@ -1508,7 +1509,7 @@ private void transferToMaster() {
             if (Config.enable_check_compatibility_mode) {
                 String msg = "check metadata compatibility successfully";
                 LOG.info(msg);
-                System.out.println(msg);
+                LogUtils.stdout(msg);
                 System.exit(0);
             }
 
@@ -1612,7 +1613,7 @@ private void transferToMaster() {
             checkLowerCaseTableNames();
 
             String msg = "master finished to replay journal, can write now.";
-            Util.stdoutWithTime(msg);
+            LogUtils.stdout(msg);
             LOG.info(msg);
             // for master, there are some new thread pools need to register metric
             ThreadPoolManager.registerAllThreadPoolMetric();
@@ -2721,7 +2722,7 @@ public void notifyNewFETypeTransfer(FrontendNodeType newType) {
         try {
             String msg = "notify new FE type transfer: " + newType;
             LOG.warn(msg);
-            Util.stdoutWithTime(msg);
+            LogUtils.stdout(msg);
             this.typeTransferQueue.put(newType);
         } catch (InterruptedException e) {
             LOG.error("failed to put new FE type: {}", newType, e);
@@ -2739,7 +2740,7 @@ protected synchronized void runOneCycle() {
                         newType = typeTransferQueue.take();
                     } catch (InterruptedException e) {
                         LOG.error("got exception when take FE type from queue", e);
-                        Util.stdoutWithTime("got exception when take FE type from queue. " + e.getMessage());
+                        LogUtils.stdout("got exception when take FE type from queue. " + e.getMessage());
                         System.exit(-1);
                     }
                     Preconditions.checkNotNull(newType);
@@ -2821,7 +2822,7 @@ protected synchronized void runOneCycle() {
                             // exit if master changed to any other type
                             String msg = "transfer FE type from MASTER to " + newType.name() + ". exit";
                             LOG.error(msg);
-                            Util.stdoutWithTime(msg);
+                            LogUtils.stdout(msg);
                             System.exit(-1);
                             break;
                         }
@@ -6320,13 +6321,13 @@ public void checkReadyOrThrowTException() throws TException {
     private void replayJournalsAndExit() {
         replayJournal(-1);
         LOG.info("check metadata compatibility successfully");
-        System.out.println("check metadata compatibility successfully");
+        LogUtils.stdout("check metadata compatibility successfully");
 
         if (Config.checkpoint_after_check_compatibility) {
             String imagePath = dumpImage();
             String msg = "the new image file path is: " + imagePath;
             LOG.info(msg);
-            System.out.println(msg);
+            LogUtils.stdout(msg);
         }
 
         System.exit(0);