From 5f0fc0d4e9db10cb77bc458e2abdfeeadc3eb262 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 30 May 2024 18:54:28 +0800 Subject: [PATCH 01/14] 1 --- .../src/main/java/org/apache/doris/common/Log4jConfig.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 8c840a876bb69a..791a59ff296b5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -60,7 +60,7 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append("\n\n") .append("\n") .append(" \n") - .append(" ") + .append(" \n") .append(" \n") .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n\n") .append(" \n") @@ -122,8 +122,8 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") - .append(" \n") + .append(" \n") + .append(" \n") .append(" \n") .append(" \n") .append(" \n") From cfca443b44b06460595d6e3277dfdf1e91ce9508 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 30 May 2024 19:25:58 +0800 Subject: [PATCH 02/14] 1 --- .../java/org/apache/doris/common/Log4jConfig.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 791a59ff296b5e..3a75be1f8b52ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -54,6 +54,12 @@ private static void getXmlConfByStrategy(final String size, final String age) { } } + private static final String LOG_PATTERN + = "%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n"; + private static final String AUDIT_LOG_PATTERN + = "%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n"; + + // @formatter:off static { // CHECKSTYLE OFF xmlConfTemplateBuilder.append("\n") @@ -62,12 +68,12 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n\n") + .append(" " + LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n\n") + .append(" " + LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") @@ -85,7 +91,7 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n\n") + .append(" " + LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") @@ -103,7 +109,7 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n\n") + .append(" " + AUDIT_LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") @@ -134,6 +140,7 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(""); // CHECKSTYLE ON } + // @formatter:on private static StrSubstitutor strSub; private static String sysLogLevel; From c1550b05b25d1474205e0cc8d3d2671d2f15a89a Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 30 May 2024 23:33:56 +0800 Subject: [PATCH 03/14] 1 --- bin/start_fe.sh | 20 +++-- fe/fe-core/pom.xml | 4 + .../org/apache/doris/common/Log4jConfig.java | 89 ++++++++++++++----- fe/pom.xml | 5 ++ 4 files changed, 87 insertions(+), 31 deletions(-) diff --git a/bin/start_fe.sh b/bin/start_fe.sh index 9d51648ff4fd66..6399943945a41b 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -176,6 +176,12 @@ if [[ ! -d "${LOG_DIR}" ]]; then mkdir -p "${LOG_DIR}" fi +STDOUT_LOGGER="${LOG_DIR}/fe.out" +if [[ "${RUN_CONSOLE}" -eq 1 ]]; then + STDOUT_LOGGER="/dev/stdout" +fi + + # check java version and choose correct JAVA_OPTS java_version="$( set -e @@ -183,16 +189,16 @@ java_version="$( )" if [[ "${java_version}" -eq 17 ]]; then if [[ -z "${JAVA_OPTS_FOR_JDK_17}" ]]; then - echo "JAVA_OPTS_FOR_JDK_17 is not set in fe.conf" >>"${LOG_DIR}/fe.out" + echo "JAVA_OPTS_FOR_JDK_17 is not set in fe.conf" >>"${STDOUT_LOGGER}" exit 1 fi final_java_opt="${JAVA_OPTS_FOR_JDK_17}" else - echo "ERROR: The jdk_version is ${java_version}, must be 17." >>"${LOG_DIR}/fe.out" + echo "ERROR: The jdk_version is ${java_version}, must be 17." >>"${STDOUT_LOGGER}" exit 1 fi -echo "Using Java version ${java_version}" >>"${LOG_DIR}/fe.out" -echo "${final_java_opt}" >>"${LOG_DIR}/fe.out" +echo "Using Java version ${java_version}" >>"${STDOUT_LOGGER}" +echo "${final_java_opt}" >>"${STDOUT_LOGGER}" # add libs to CLASSPATH DORIS_FE_JAR= @@ -239,7 +245,7 @@ if [[ -n "${JACOCO_COVERAGE_OPT}" ]]; then coverage_opt="${JACOCO_COVERAGE_OPT}" fi -date >>"${LOG_DIR}/fe.out" +date >>"${STDOUT_LOGGER}" if [[ "${HELPER}" != "" ]]; then # change it to '-helper' to be compatible with code in Frontend @@ -253,12 +259,12 @@ if [[ "${IMAGE_TOOL}" -eq 1 ]]; then echo "Internal error, USE IMAGE_TOOL like: ./start_fe.sh --image image_path" fi elif [[ "${RUN_DAEMON}" -eq 1 ]]; then - nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" ${coverage_opt:+${coverage_opt}} org.apache.doris.DorisFE ${HELPER:+${HELPER}} "${METADATA_FAILURE_RECOVERY}" "$@" >>"${LOG_DIR}/fe.out" 2>&1 >"${STDOUT_LOGGER}" 2>&1 >"${LOG_DIR}/fe.out" 2>&1 >"${STDOUT_LOGGER}" 2>&1 log4j-web ${log4j2.version} + + org.apache.logging.log4j + log4j-iostreams + org.springframework.boot spring-boot-devtools diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 3a75be1f8b52ff..22c6fc555a0ba3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -21,15 +21,18 @@ import com.google.common.base.Strings; import com.google.common.collect.Maps; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.ConfigurationSource; import org.apache.logging.log4j.core.config.xml.XmlConfiguration; import org.apache.logging.log4j.core.lookup.Interpolator; import org.apache.logging.log4j.core.lookup.StrSubstitutor; +import org.apache.logging.log4j.io.IoBuilder; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -54,10 +57,31 @@ private static void getXmlConfByStrategy(final String size, final String age) { } } - private static final String LOG_PATTERN - = "%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n"; + // Placeholders + private static final String RUNTIME_LOG_FORMAT_PLACEHOLDER = ""; + private static final String VERBOSE_MODULE_PLACEHOLDER = ""; + private static final String CONSOLE_LOGGER_PLACEHOLDER = ""; + private static final String AUDIT_CONSOLE_LOGGER_PLACEHOLDER = ""; + private static final String RUNTIME_LOG_MARKER_PLACEHOLDER = ""; + private static final String AUDIT_LOG_MARKER_PLACEHOLDER = ""; + + // Appender names + private static final String RUNTIME_LOG_CONSOLE_APPENDER = "Console"; + private static final String RUNTIME_LOG_FILE_APPENDER = "Sys"; + private static final String RUNTIME_LOG_WARN_FILE_APPENDER = "SysWF"; + private static final String AUDIT_LOG_CONSOLE_APPENDER = "AuditConsole"; + private static final String AUDIT_LOG_FILE_APPENDER = "AuditFile"; + + // Log patterns + private static final String RUNTIME_LOG_PATTERN + = RUNTIME_LOG_MARKER_PLACEHOLDER + "%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)" + + RUNTIME_LOG_FORMAT_PLACEHOLDER; private static final String AUDIT_LOG_PATTERN - = "%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n"; + = AUDIT_LOG_MARKER_PLACEHOLDER + "%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n"; + + // Log markers + private static final String RUNTIME_LOG_MARKER = "RuntimeLogger"; + private static final String AUDIT_LOG_MARKER = "AuditLogger"; // @formatter:off static { @@ -66,14 +90,19 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append("\n\n") .append("\n") .append(" \n") - .append(" \n") + .append(" \n") + .append(" \n") + .append(" " + RUNTIME_LOG_PATTERN + "\n") + .append(" \n") + .append(" \n") + .append(" \n") .append(" \n") - .append(" " + LOG_PATTERN + "\n") + .append(" " + AUDIT_LOG_PATTERN + "\n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" \n") - .append(" " + LOG_PATTERN + "\n") + .append(" " + RUNTIME_LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") @@ -89,9 +118,9 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" \n") - .append(" " + LOG_PATTERN + "\n") + .append(" " + RUNTIME_LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") @@ -107,7 +136,7 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" \n") .append(" " + AUDIT_LOG_PATTERN + "\n") .append(" \n") @@ -128,14 +157,15 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") - .append(" \n") - .append(" \n") + .append(" \n") + .append(" \n") + .append(" " + CONSOLE_LOGGER_PLACEHOLDER + "\n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") + .append(" " + AUDIT_CONSOLE_LOGGER_PLACEHOLDER + "\n") .append(" \n") - .append(" \n") + .append(" " + VERBOSE_MODULE_PLACEHOLDER + "\n") .append(" \n") .append(""); // CHECKSTYLE ON @@ -217,24 +247,24 @@ private static void reconfig() throws IOException { for (String s : auditModules) { sb.append(""); } - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", - sb.toString()); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(VERBOSE_MODULE_PLACEHOLDER, sb.toString()); if (sysLogMode.equalsIgnoreCase("NORMAL")) { - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", - " [%C{1}.%M():%L] "); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_FORMAT_PLACEHOLDER, " [%C{1}.%M():%L] "); } else { - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", " "); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_FORMAT_PLACEHOLDER, " "); if (sysLogMode.equalsIgnoreCase("ASYNC")) { newXmlConfTemplate = newXmlConfTemplate.replaceAll("Root", "AsyncRoot"); } } if (foreground) { - StringBuilder consoleLogger = new StringBuilder(); - consoleLogger.append("\n"); - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", - consoleLogger.toString()); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_MARKER_PLACEHOLDER, RUNTIME_LOG_MARKER); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(AUDIT_LOG_MARKER_PLACEHOLDER, AUDIT_LOG_MARKER); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(CONSOLE_LOGGER_PLACEHOLDER, + "\n"); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(AUDIT_CONSOLE_LOGGER_PLACEHOLDER, + "\n"); } Map properties = Maps.newHashMap(); @@ -284,6 +314,17 @@ private static void reconfig() throws IOException { } catch (Exception e) { throw new IOException("Error occurred while configuring Log4j", e); } + + redirectStd(); + } + + private static void redirectStd() { + PrintStream logPrintStream = IoBuilder.forLogger(LogManager.getLogger("system.out")).setLevel(Level.INFO) + .buildPrintStream(); + System.setOut(logPrintStream); + PrintStream errorPrintStream = IoBuilder.forLogger(LogManager.getLogger("system.err")).setLevel(Level.ERROR) + .buildPrintStream(); + System.setErr(errorPrintStream); } public static String getLogXmlConfTemplate() { diff --git a/fe/pom.xml b/fe/pom.xml index 2eaa31c43cd814..c443a07b001dc1 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -824,6 +824,11 @@ under the License. log4j-core ${log4j2.version} + + org.apache.logging.log4j + log4j-iostreams + ${log4j2.version} + org.apache.logging.log4j log4j-slf4j-impl From 764995804b47a41cee4c74ee7327e43116251047 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 30 May 2024 23:53:44 +0800 Subject: [PATCH 04/14] 2 --- bin/start_fe.sh | 21 ++++++++++++------- .../org/apache/doris/common/Log4jConfig.java | 4 ++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/bin/start_fe.sh b/bin/start_fe.sh index 6399943945a41b..6e76ad220d0def 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -177,9 +177,13 @@ if [[ ! -d "${LOG_DIR}" ]]; then fi STDOUT_LOGGER="${LOG_DIR}/fe.out" -if [[ "${RUN_CONSOLE}" -eq 1 ]]; then - STDOUT_LOGGER="/dev/stdout" -fi +log() { + if [[ "${RUN_CONSOLE}" -eq 1 ]]; then + echo "$1" + else + echo "$1" >> "${STDOUT_LOGGER}" + fi +} # check java version and choose correct JAVA_OPTS @@ -189,16 +193,16 @@ java_version="$( )" if [[ "${java_version}" -eq 17 ]]; then if [[ -z "${JAVA_OPTS_FOR_JDK_17}" ]]; then - echo "JAVA_OPTS_FOR_JDK_17 is not set in fe.conf" >>"${STDOUT_LOGGER}" + log "JAVA_OPTS_FOR_JDK_17 is not set in fe.conf" exit 1 fi final_java_opt="${JAVA_OPTS_FOR_JDK_17}" else - echo "ERROR: The jdk_version is ${java_version}, must be 17." >>"${STDOUT_LOGGER}" + log "ERROR: The jdk_version is ${java_version}, must be 17." exit 1 fi -echo "Using Java version ${java_version}" >>"${STDOUT_LOGGER}" -echo "${final_java_opt}" >>"${STDOUT_LOGGER}" +log "Using Java version ${java_version}" +log "${final_java_opt}" # add libs to CLASSPATH DORIS_FE_JAR= @@ -245,7 +249,8 @@ if [[ -n "${JACOCO_COVERAGE_OPT}" ]]; then coverage_opt="${JACOCO_COVERAGE_OPT}" fi -date >>"${STDOUT_LOGGER}" +CUR_DATE=`date` +log "start time: ${CUR_DATE}" if [[ "${HELPER}" != "" ]]; then # change it to '-helper' to be compatible with code in Frontend diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 22c6fc555a0ba3..ab0d2dfe779493 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -80,8 +80,8 @@ private static void getXmlConfByStrategy(final String size, final String age) { = AUDIT_LOG_MARKER_PLACEHOLDER + "%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n"; // Log markers - private static final String RUNTIME_LOG_MARKER = "RuntimeLogger"; - private static final String AUDIT_LOG_MARKER = "AuditLogger"; + private static final String RUNTIME_LOG_MARKER = "RuntimeLogger "; + private static final String AUDIT_LOG_MARKER = "AuditLogger "; // @formatter:off static { From 5b1b38d36e6a2408638860ede565526e843a7dfd Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 31 May 2024 00:09:16 +0800 Subject: [PATCH 05/14] 3 --- .../src/main/java/org/apache/doris/common/Log4jConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index ab0d2dfe779493..7d64ed290b6eee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -75,7 +75,7 @@ private static void getXmlConfByStrategy(final String size, final String age) { // Log patterns private static final String RUNTIME_LOG_PATTERN = RUNTIME_LOG_MARKER_PLACEHOLDER + "%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)" - + RUNTIME_LOG_FORMAT_PLACEHOLDER; + + RUNTIME_LOG_FORMAT_PLACEHOLDER + "%m%n"; private static final String AUDIT_LOG_PATTERN = AUDIT_LOG_MARKER_PLACEHOLDER + "%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n"; From 6ff43f14703c39ff361af684137bbcc5c48bba68 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 31 May 2024 00:19:04 +0800 Subject: [PATCH 06/14] 4 --- .../java/org/apache/doris/common/Config.java | 7 ++++++ .../main/java/org/apache/doris/DorisFE.java | 3 --- .../org/apache/doris/common/Log4jConfig.java | 24 ++++++++++++++----- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 8420e598d2b661..f165aec79e5756 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -127,6 +127,13 @@ public class Config extends ConfigBase { @ConfField(description = {"是否压缩 FE 的 Audit 日志", "enable compression for FE audit log file"}) public static boolean audit_log_enable_compress = false; + @ConfField(description = {"是否使用文件记录日志。当使用 --console 启动 FE 时，全部日志会写入到标准输出。" + + "此时关闭关闭这个选项，不再使用文件记录日志。", + "Whether to use file to record log. When starting FE with --console, " + + "all logs will be written to standard output. " + + "At this time, close this option and no longer use file to record log."}) + public static boolean enable_file_logger = true; + @ConfField(mutable = false, masterOnly = false, description = {"是否检查table锁泄漏", "Whether to check table lock leaky"}) public static boolean check_table_lock_leaky = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java index 583f20687c69f4..f614447b3eb71f 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java @@ -481,9 +481,7 @@ private static void tryLockProcess() { + "same time"); } - private static void releaseFileLockAndCloseFileChannel() { - if (processFileLock != null && processFileLock.isValid()) { try { processFileLock.release(); @@ -498,7 +496,6 @@ private static void releaseFileLockAndCloseFileChannel() { LOG.warn("release process lock file failed", ignored); } } - } public static void overwriteConfigs() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 7d64ed290b6eee..452c31e9fa6231 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -60,8 +60,11 @@ private static void getXmlConfByStrategy(final String size, final String age) { // Placeholders private static final String RUNTIME_LOG_FORMAT_PLACEHOLDER = ""; private static final String VERBOSE_MODULE_PLACEHOLDER = ""; - private static final String CONSOLE_LOGGER_PLACEHOLDER = ""; + private static final String CONSOLE_APPENDER_PLACEHOLDER = ""; + private static final String RUNTIME_LOG_FILE_APPENDER_PLACEHOLDER = ""; + private static final String RUNTIME_LOG_WARN_FILE_APPENDER_PLACEHOLDER = ""; private static final String AUDIT_CONSOLE_LOGGER_PLACEHOLDER = ""; + private static final String AUDIT_FILE_LOGGER_PLACEHOLDER = ""; private static final String RUNTIME_LOG_MARKER_PLACEHOLDER = ""; private static final String AUDIT_LOG_MARKER_PLACEHOLDER = ""; @@ -157,12 +160,12 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") - .append(" \n") - .append(" " + CONSOLE_LOGGER_PLACEHOLDER + "\n") + .append(" " + RUNTIME_LOG_FILE_APPENDER_PLACEHOLDER + "\n") + .append(" " + RUNTIME_LOG_WARN_FILE_APPENDER_PLACEHOLDER + "\n") + .append(" " + CONSOLE_APPENDER_PLACEHOLDER + "\n") .append(" \n") .append(" \n") - .append(" \n") + .append(" " + AUDIT_FILE_LOGGER_PLACEHOLDER + "\n") .append(" " + AUDIT_CONSOLE_LOGGER_PLACEHOLDER + "\n") .append(" \n") .append(" " + VERBOSE_MODULE_PLACEHOLDER + "\n") @@ -258,10 +261,19 @@ private static void reconfig() throws IOException { } } + if (Config.enable_file_logger) { + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_FILE_APPENDER_PLACEHOLDER, + "\n"); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_WARN_FILE_APPENDER_PLACEHOLDER, + "\n"); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(AUDIT_FILE_LOGGER_PLACEHOLDER, + "\n"); + } + if (foreground) { newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_MARKER_PLACEHOLDER, RUNTIME_LOG_MARKER); newXmlConfTemplate = newXmlConfTemplate.replaceAll(AUDIT_LOG_MARKER_PLACEHOLDER, AUDIT_LOG_MARKER); - newXmlConfTemplate = newXmlConfTemplate.replaceAll(CONSOLE_LOGGER_PLACEHOLDER, + newXmlConfTemplate = newXmlConfTemplate.replaceAll(CONSOLE_APPENDER_PLACEHOLDER, "\n"); newXmlConfTemplate = newXmlConfTemplate.replaceAll(AUDIT_CONSOLE_LOGGER_PLACEHOLDER, "\n"); From 2f05156d1d6e5258e49d05318ab43b28b02541dc Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 31 May 2024 00:33:10 +0800 Subject: [PATCH 07/14] 3 --- bin/start_fe.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/start_fe.sh b/bin/start_fe.sh index 6e76ad220d0def..64c348cb64e7ed 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -181,11 +181,10 @@ log() { if [[ "${RUN_CONSOLE}" -eq 1 ]]; then echo "$1" else - echo "$1" >> "${STDOUT_LOGGER}" + echo "$1" >>"${STDOUT_LOGGER}" fi } - # check java version and choose correct JAVA_OPTS java_version="$( set -e @@ -249,7 +248,7 @@ if [[ -n "${JACOCO_COVERAGE_OPT}" ]]; then coverage_opt="${JACOCO_COVERAGE_OPT}" fi -CUR_DATE=`date` +CUR_DATE=$(date) log "start time: ${CUR_DATE}" if [[ "${HELPER}" != "" ]]; then From c6bb706b5d7c3bb255389760a5143abaa2361481 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Jun 2024 10:56:59 +0800 Subject: [PATCH 08/14] remove stdout --- .../org/apache/doris/common/LogUtils.java | 44 +++++++++++++++++++ .../main/java/org/apache/doris/DorisFE.java | 19 ++++---- .../java/org/apache/doris/catalog/Env.java | 15 ++++--- .../org/apache/doris/common/Log4jConfig.java | 4 +- .../doris/common/util/DocGenerator.java | 9 ++-- .../apache/doris/common/util/JdkUtils.java | 3 +- .../doris/common/util/PlatformName.java | 33 -------------- .../org/apache/doris/common/util/Util.java | 4 -- .../lakesoul/LakeSoulExternalTable.java | 1 - .../doris/ha/BDBStateChangeListener.java | 4 +- .../doris/journal/bdbje/BDBEnvironment.java | 3 +- .../doris/journal/bdbje/BDBJEJournal.java | 14 +++--- .../doris/journal/bdbje/BDBJournalCursor.java | 2 +- .../apache/doris/journal/bdbje/BDBTool.java | 27 ++++++------ .../doris/journal/local/LocalJournal.java | 2 +- .../journal/local/LocalJournalCursor.java | 4 +- .../jobs/rewrite/CustomRewriteJob.java | 5 --- .../org/apache/doris/persist/EditLog.java | 3 +- 18 files changed, 99 insertions(+), 97 deletions(-) create mode 100644 fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/util/PlatformName.java diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java new file mode 100644 index 00000000000000..a7df881e928836 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java @@ -0,0 +1,44 @@ +/* + * // Licensed to the Apache Software Foundation (ASF) under one + * // or more contributor license agreements. See the NOTICE file + * // distributed with this work for additional information + * // regarding copyright ownership. The ASF licenses this file + * // to you under the Apache License, Version 2.0 (the + * // "License"); you may not use this file except in compliance + * // with the License. You may obtain a copy of the License at + * // + * // http://www.apache.org/licenses/LICENSE-2.0 + * // + * // Unless required by applicable law or agreed to in writing, + * // software distributed under the License is distributed on an + * // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * // KIND, either express or implied. See the License for the + * // specific language governing permissions and limitations + * // under the License. + */ + +package org.apache.doris.common; + +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +public class LogUtils { + + public static final String STDOUT_LOG_MARKER = "StdoutLogger "; + public static final String STDERR_LOG_MARKER = "StderrLogger "; + + private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS"); + + private static String formattedTime() { + LocalDateTime dateTime = LocalDateTime.now(); + return dateTime.format(TIME_FORMATTER); + } + + public static void stdout(String message) { + System.out.println(STDOUT_LOG_MARKER + formattedTime() + " " + message); + } + + public static void stderr(String message) { + System.err.println(STDERR_LOG_MARKER + formattedTime() + " " + message); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java index f614447b3eb71f..3ca39bbfd0690e 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java @@ -24,6 +24,7 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.LdapConfig; import org.apache.doris.common.Log4jConfig; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.Version; import org.apache.doris.common.util.JdkUtils; @@ -386,11 +387,11 @@ private static CommandLineOptions parseArgs(String[] args) { } private static void printVersion() { - System.out.println("Build version: " + Version.DORIS_BUILD_VERSION); - System.out.println("Build time: " + Version.DORIS_BUILD_TIME); - System.out.println("Build info: " + Version.DORIS_BUILD_INFO); - System.out.println("Build hash: " + Version.DORIS_BUILD_HASH); - System.out.println("Java compile version: " + Version.DORIS_JAVA_COMPILE_VERSION); + LogUtils.stdout("Build version: " + Version.DORIS_BUILD_VERSION); + LogUtils.stdout("Build time: " + Version.DORIS_BUILD_TIME); + LogUtils.stdout("Build info: " + Version.DORIS_BUILD_INFO); + LogUtils.stdout("Build hash: " + Version.DORIS_BUILD_HASH); + LogUtils.stdout("Java compile version: " + Version.DORIS_JAVA_COMPILE_VERSION); LOG.info("Build version: {}", Version.DORIS_BUILD_VERSION); LOG.info("Build time: {}", Version.DORIS_BUILD_TIME); @@ -413,16 +414,16 @@ private static void checkCommandLineOptions(CommandLineOptions cmdLineOpts) { } else if (cmdLineOpts.runImageTool()) { File imageFile = new File(cmdLineOpts.getImagePath()); if (!imageFile.exists()) { - System.out.println("image does not exist: " + imageFile.getAbsolutePath() + LogUtils.stderr("image does not exist: " + imageFile.getAbsolutePath() + " . Please put an absolute path instead"); System.exit(-1); } else { - System.out.println("Start to load image: "); + LogUtils.stdout("Start to load image: "); try { MetaReader.read(imageFile, Env.getCurrentEnv()); - System.out.println("Load image success. Image file " + cmdLineOpts.getImagePath() + " is valid"); + LogUtils.stdout("Load image success. Image file " + cmdLineOpts.getImagePath() + " is valid"); } catch (Exception e) { - System.out.println("Load image failed. Image file " + cmdLineOpts.getImagePath() + " is invalid"); + LogUtils.stderr("Load image failed. Image file " + cmdLineOpts.getImagePath() + " is invalid"); LOG.warn("", e); } finally { System.exit(0); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index d49e7dd73b27dc..a91ab2ea5ba434 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -103,6 +103,7 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.NereidsSqlCacheManager; import org.apache.doris.common.Pair; @@ -1508,7 +1509,7 @@ private void transferToMaster() { if (Config.enable_check_compatibility_mode) { String msg = "check metadata compatibility successfully"; LOG.info(msg); - System.out.println(msg); + LogUtils.stdout(msg); System.exit(0); } @@ -1612,7 +1613,7 @@ private void transferToMaster() { checkLowerCaseTableNames(); String msg = "master finished to replay journal, can write now."; - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); LOG.info(msg); // for master, there are some new thread pools need to register metric ThreadPoolManager.registerAllThreadPoolMetric(); @@ -2721,7 +2722,7 @@ public void notifyNewFETypeTransfer(FrontendNodeType newType) { try { String msg = "notify new FE type transfer: " + newType; LOG.warn(msg); - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); this.typeTransferQueue.put(newType); } catch (InterruptedException e) { LOG.error("failed to put new FE type: {}", newType, e); @@ -2739,7 +2740,7 @@ protected synchronized void runOneCycle() { newType = typeTransferQueue.take(); } catch (InterruptedException e) { LOG.error("got exception when take FE type from queue", e); - Util.stdoutWithTime("got exception when take FE type from queue. " + e.getMessage()); + LogUtils.stdout("got exception when take FE type from queue. " + e.getMessage()); System.exit(-1); } Preconditions.checkNotNull(newType); @@ -2821,7 +2822,7 @@ protected synchronized void runOneCycle() { // exit if master changed to any other type String msg = "transfer FE type from MASTER to " + newType.name() + ". exit"; LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); System.exit(-1); break; } @@ -6320,13 +6321,13 @@ public void checkReadyOrThrowTException() throws TException { private void replayJournalsAndExit() { replayJournal(-1); LOG.info("check metadata compatibility successfully"); - System.out.println("check metadata compatibility successfully"); + LogUtils.stdout("check metadata compatibility successfully"); if (Config.checkpoint_after_check_compatibility) { String imagePath = dumpImage(); String msg = "the new image file path is: " + imagePath; LOG.info(msg); - System.out.println(msg); + LogUtils.stdout(msg); } System.exit(0); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 452c31e9fa6231..5395de076ae223 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -305,9 +305,7 @@ private static void reconfig() throws IOException { strSub = new StrSubstitutor(new Interpolator(properties)); newXmlConfTemplate = strSub.replace(newXmlConfTemplate); - System.out.println("====="); - System.out.println(newXmlConfTemplate); - System.out.println("====="); + LogUtils.stdout("=====\n" + newXmlConfTemplate + "\n====="); logXmlConfTemplate = newXmlConfTemplate; SpringLog4j2Config.writeSpringLogConf(customConfDir); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java index 0a207925fde846..c0a544d825dcce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java @@ -19,6 +19,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ConfigBase.ConfField; +import org.apache.doris.common.LogUtils; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.VariableMgr; @@ -148,7 +149,7 @@ private String genFEConfigDoc(Lang lang) throws IllegalAccessException { sortedDoc.put(field.getName(), res); } } catch (Exception e) { - System.out.println("Failed to generate doc for field: " + field.getName()); + LogUtils.stderr("Failed to generate doc for field: " + field.getName()); throw e; } } @@ -204,7 +205,7 @@ private String genSessionVariableDoc(Lang lang) throws IllegalAccessException { sortedDoc.put(field.getAnnotation(VariableMgr.VarAttr.class).name(), res); } } catch (Exception e) { - System.out.println("Failed to generate doc for " + field.getName()); + LogUtils.stderr("Failed to generate doc for " + field.getName()); throw e; } } @@ -217,7 +218,7 @@ private String genSessionVariableDoc(Lang lang) throws IllegalAccessException { sortedDoc.put(field.getAnnotation(VariableMgr.VarAttr.class).name(), res); } } catch (Exception e) { - System.out.println("Failed to generate doc for field: " + field.getName()); + LogUtils.stderr("Failed to generate doc for field: " + field.getName()); throw e; } } @@ -288,7 +289,7 @@ public static void main(String[] args) { sessionVariableDocOutputPath, sessionVariableDocOutputPathCN); try { docGenerator.generate(); - System.out.println("Done!"); + LogUtils.stdout("Done!"); } catch (Exception e) { log.info("failed to generate doc", e); System.exit(-1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java index 34a604eb0d3b76..7d3a0402bcbbb3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java @@ -18,6 +18,7 @@ package org.apache.doris.common.util; import org.apache.doris.common.Config; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.Version; import com.google.common.annotations.VisibleForTesting; @@ -42,7 +43,7 @@ public static boolean checkJavaVersion() { int runtimeVersion = JdkUtils.getJavaVersionAsInteger(javaRuntimeVersionStr); if (runtimeVersion < compileVersion) { - System.out.println("The runtime java version " + javaRuntimeVersionStr + " is less than " + LogUtils.stdout("The runtime java version " + javaRuntimeVersionStr + " is less than " + "compile version " + javaCompileVersionStr); return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PlatformName.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PlatformName.java deleted file mode 100644 index 0953c37196d2ee..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PlatformName.java +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.util; - -public class PlatformName { - - private static final String platformName = System.getProperty("os.name") + "-" - + System.getProperty("os.arch") + "-" - + System.getProperty("sun.arch.data.model"); - - public static String getPlatformName() { - return platformName; - } - - public static void main(String[] args) { - System.out.println(platformName); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index 4e4c6a947359fb..e73df139e01a8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -424,10 +424,6 @@ public static boolean getBooleanPropertyOrDefault(String valStr, boolean default } } - public static void stdoutWithTime(String msg) { - System.out.println("[" + TimeUtils.longToTimeString(System.currentTimeMillis()) + "] " + msg); - } - // not support encode negative value now public static void encodeVarint64(long source, DataOutput out) throws IOException { assert source >= 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java index 5ba686fc6882b3..46e8d1db47c723 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java @@ -149,7 +149,6 @@ public Optional initSchema() { TableInfo tableInfo = ((LakeSoulExternalCatalog) catalog).getLakeSoulTable(dbName, name); String tableSchema = tableInfo.getTableSchema(); DBUtil.TablePartitionKeys partitionKeys = DBUtil.parseTableInfoPartitions(tableInfo.getPartitions()); - System.out.println(tableSchema); Schema schema; try { schema = Schema.fromJSON(tableSchema); diff --git a/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java b/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java index 2ff66a5aad0162..302717c6050e8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java +++ b/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java @@ -18,7 +18,7 @@ package org.apache.doris.ha; import org.apache.doris.catalog.Env; -import org.apache.doris.common.util.Util; +import org.apache.doris.common.LogUtils; import com.google.common.base.Preconditions; import com.sleepycat.je.rep.StateChangeEvent; @@ -57,7 +57,7 @@ public synchronized void stateChange(StateChangeEvent sce) throws RuntimeExcepti default: { String msg = "this node is " + sce.getState().name(); LOG.warn(msg); - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); return; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java index 47a41b8226ad01..d2784a05e70c7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java @@ -105,8 +105,7 @@ public void setup(File envHome, String selfNodeName, String selfNodeHostPort, DbResetRepGroup resetUtility = new DbResetRepGroup( envHome, PALO_JOURNAL_GROUP, selfNodeName, selfNodeHostPort); resetUtility.reset(); - LOG.warn("WARNING: metadata recovery mode, group has been reset."); - System.out.println("WARNING: metadata recovery mode, group has been reset."); + LOG.warn("metadata recovery mode, group has been reset."); } // set replication config diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java index 0d342a986cd40f..9e78b5eb47a14b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java @@ -20,10 +20,10 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.io.DataOutputBuffer; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.NetUtils; -import org.apache.doris.common.util.Util; import org.apache.doris.journal.Journal; import org.apache.doris.journal.JournalBatch; import org.apache.doris.journal.JournalCursor; @@ -120,7 +120,7 @@ public synchronized void rollJournal() { + "journal id: %d, current db: %s, expected db count: %d", newName, currentDbName, newNameVerify); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } } @@ -193,7 +193,7 @@ public synchronized long write(JournalBatch batch) throws IOException { String msg = "write bdb failed. will exit. the first journalId: " + firstId + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } catch (DatabaseException e) { LOG.error("catch an exception when writing to database. sleep and retry. the first journal id {}", @@ -218,7 +218,7 @@ public synchronized long write(JournalBatch batch) throws IOException { String msg = "write bdb failed. will exit. the first journalId: " + firstId + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); return 0; // unreachable! } @@ -285,7 +285,7 @@ public synchronized long write(short op, Writable writable) throws IOException { String msg = "write bdb failed. will exit. journalId: " + id + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } catch (DatabaseException e) { LOG.error("catch an exception when writing to database. sleep and retry. journal id {}", id, e); @@ -301,7 +301,7 @@ public synchronized long write(short op, Writable writable) throws IOException { String msg = "write bdb failed. will exit. journalId: " + id + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } return id; @@ -357,7 +357,7 @@ public JournalEntity read(long journalId) { LOG.warn("", e); } } else { - System.out.println("No record found for key '" + journalId + "'."); + LOG.warn("No record found for key '{}'.", journalId); } } catch (Exception e) { LOG.warn("catch an exception when get JournalEntity. key:{}", journalId, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java index 5466df71ddcc8a..8c48cab42c5df6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java @@ -46,7 +46,7 @@ public class BDBJournalCursor implements JournalCursor { public static BDBJournalCursor getJournalCursor(BDBEnvironment env, long fromKey, long toKey) { if (toKey < fromKey || fromKey < 0) { - System.out.println("Invalid key range!"); + LOG.warn("Invalid key range! fromKey:{} toKey:{}", fromKey, toKey); return null; } BDBJournalCursor cursor = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java index cc9defa378bf57..55c15475282c36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java @@ -18,6 +18,7 @@ package org.apache.doris.journal.bdbje; import org.apache.doris.catalog.Env; +import org.apache.doris.common.LogUtils; import org.apache.doris.journal.JournalEntity; import org.apache.doris.meta.MetaContext; @@ -66,7 +67,7 @@ public boolean run() { env = new Environment(new File(metaPath), envConfig); } catch (DatabaseException e) { LOG.warn("", e); - System.err.println("Failed to open BDBJE env: " + Env.getCurrentEnv().getBdbDir() + ". exit"); + LogUtils.stderr("Failed to open BDBJE env: " + Env.getCurrentEnv().getBdbDir() + ". exit"); return false; } Preconditions.checkNotNull(env); @@ -75,7 +76,7 @@ public boolean run() { if (options.isListDbs()) { // list all databases List dbNames = env.getDatabaseNames(); - System.out.println(JSONArray.toJSONString(dbNames)); + LogUtils.stdout(JSONArray.toJSONString(dbNames)); return true; } else { // db operations @@ -90,7 +91,7 @@ public boolean run() { // get db stat Map statMap = Maps.newHashMap(); statMap.put("count", String.valueOf(db.count())); - System.out.println(JSONObject.toJSONString(statMap)); + LogUtils.stdout(JSONObject.toJSONString(statMap)); return true; } else { // set from key @@ -99,7 +100,7 @@ public boolean run() { try { fromKey = Long.valueOf(fromKeyStr); } catch (NumberFormatException e) { - System.err.println("Not a valid from key: " + fromKeyStr); + LogUtils.stderr("Not a valid from key: " + fromKeyStr); return false; } @@ -109,13 +110,13 @@ public boolean run() { try { endKey = Long.valueOf(options.getEndKey()); } catch (NumberFormatException e) { - System.err.println("Not a valid end key: " + options.getEndKey()); + LogUtils.stderr("Not a valid end key: " + options.getEndKey()); return false; } } if (fromKey > endKey) { - System.err.println("from key should less than or equal to end key[" + LogUtils.stderr("from key should less than or equal to end key[" + fromKey + " vs. " + endKey + "]"); return false; } @@ -132,7 +133,7 @@ public boolean run() { } } catch (Exception e) { LOG.warn("", e); - System.err.println("Failed to run bdb tools"); + LogUtils.stderr("Failed to run bdb tools"); return false; } return true; @@ -155,15 +156,15 @@ private void getValueByKey(Database db, Long key) entity.readFields(in); } catch (Exception e) { LOG.warn("", e); - System.err.println("Fail to read journal entity for key: " + key + ". reason: " + e.getMessage()); + LogUtils.stderr("Fail to read journal entity for key: " + key + ". reason: " + e.getMessage()); System.exit(-1); } - System.out.println("key: " + key); - System.out.println("op code: " + entity.getOpCode()); - System.out.println("value: " + entity.getData().toString()); + LogUtils.stdout("key: " + key); + LogUtils.stdout("op code: " + entity.getOpCode()); + LogUtils.stdout("value: " + entity.getData().toString()); } else if (status == OperationStatus.NOTFOUND) { - System.out.println("key: " + key); - System.out.println("value: NOT FOUND"); + LogUtils.stdout("key: " + key); + LogUtils.stdout("value: NOT FOUND"); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java index 45ae377a40171c..8e39f8a6a616be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java @@ -84,7 +84,7 @@ public synchronized void rollJournal() { try { storage = new Storage(imageDir); if (journalId.get() == storage.getEditsSeq()) { - System.out.println("Does not need to roll!"); + LOG.warn("Does not need to roll! journalId: {}, editsSeq: {}", journalId.get(), storage.getEditsSeq()); return; } if (outputStream != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java index d570719dd3a13d..d24e394927b1cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java @@ -50,7 +50,7 @@ public final class LocalJournalCursor implements JournalCursor { public static LocalJournalCursor getJournalCursor(String imageDir, long fromKey, long toKey) { if (toKey < fromKey && toKey != -1 || fromKey < 0) { - System.out.println("Invalid key range!"); + LOG.warn("Invalid fromKey:{} toKey:{}", fromKey, toKey); return null; } long newToKey = toKey; @@ -95,7 +95,7 @@ private LocalJournalCursor(String imageDir, long fromKey, long toKey) throws IOE } if (fileName == null) { - System.out.println("Can not find the key:" + fromKey); + LOG.warn("Can not find the key:{}", fromKey); throw new IOException(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java index 0e58f1bc976ba5..ae3aa52c4f2094 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java @@ -79,9 +79,4 @@ public void execute(JobContext context) { public boolean isOnce() { return false; } - - private void printTraceLog(RuleType ruleType, String traceBefore, String traceAfter) { - System.out.println("========== " + getClass().getSimpleName() + " " + ruleType - + " ==========\nbefore:\n" + traceBefore + "\n\nafter:\n" + traceAfter + "\n"); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index d440cf77cac5e7..d5177844868373 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -446,8 +446,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { Frontend fe = (Frontend) journal.getData(); env.replayDropFrontend(fe); if (fe.getNodeName().equals(Env.getCurrentEnv().getNodeName())) { - System.out.println("current fe " + fe + " is removed. will exit"); - LOG.info("current fe " + fe + " is removed. will exit"); + LOG.warn("current fe {} is removed. will exit", fe); System.exit(-1); } break; From adb02ec3765e51cdfbc81ac87199a083b6ce34c1 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Jun 2024 10:58:49 +0800 Subject: [PATCH 09/14] fix2 --- .../org/apache/doris/cloud/security/UrlSecurityChecker.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/cloud/security/UrlSecurityChecker.java b/fe/fe-common/src/main/java/org/apache/doris/cloud/security/UrlSecurityChecker.java index 8d4056595e17e0..5f79343d0f9bd4 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/cloud/security/UrlSecurityChecker.java +++ b/fe/fe-common/src/main/java/org/apache/doris/cloud/security/UrlSecurityChecker.java @@ -18,6 +18,7 @@ package org.apache.doris.cloud.security; import org.apache.doris.common.Config; +import org.apache.doris.common.LogUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -38,8 +39,8 @@ public class UrlSecurityChecker extends SecurityChecker { urlSecurityStopCheckMethod = clazz.getMethod("stopSSRFNetHookChecking", String.class); } catch (Exception e) { LOG.warn("security_checker_class_name:{} exception:", Config.security_checker_class_name, e); + LogUtils.stderr("Failed to find com.aliyun.securitysdk.SecurityUtil's method"); e.printStackTrace(); - System.out.println("Failed to find com.aliyun.securitysdk.SecurityUtil's method"); } } From e76b301c0cef87f8b24193d81d4b07898e3a65b5 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Jun 2024 12:18:23 +0800 Subject: [PATCH 10/14] add layout --- fe/fe-common/pom.xml | 12 ++++++ .../org/apache/doris/common/LogUtils.java | 40 +++++++++++++++++++ fe/fe-core/pom.xml | 9 ----- .../org/apache/doris/common/Log4jConfig.java | 5 ++- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml index e123e430f2ba73..458cb6ff9e4064 100644 --- a/fe/fe-common/pom.xml +++ b/fe/fe-common/pom.xml @@ -120,6 +120,18 @@ under the License. com.amazonaws aws-java-sdk-s3 + + org.apache.logging.log4j + log4j-web + + + org.apache.logging.log4j + log4j-iostreams + + + org.apache.logging.log4j + log4j-core + doris-fe-common diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java index a7df881e928836..1a18ec222cd40d 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java @@ -19,6 +19,14 @@ package org.apache.doris.common; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.layout.AbstractStringLayout; +import org.apache.logging.log4j.core.layout.PatternLayout; +import org.apache.logging.log4j.core.util.StringBuilderWriter; + +import java.io.PrintWriter; +import java.nio.charset.Charset; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -41,4 +49,36 @@ public static void stdout(String message) { public static void stderr(String message) { System.err.println(STDERR_LOG_MARKER + formattedTime() + " " + message); } + + public static class SingleLineExceptionLayout extends AbstractStringLayout { + + private final PatternLayout patternLayout; + + protected SingleLineExceptionLayout(PatternLayout patternLayout, Charset charset) { + super(charset); + this.patternLayout = patternLayout; + } + + @Override + public String toSerializable(LogEvent event) { + StringBuilder result = new StringBuilder(patternLayout.toSerializable(event)); + + if (event.getThrown() != null) { + StringBuilderWriter sw = new StringBuilderWriter(); + event.getThrown().printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString().replace("\n", " ").replace("\r", " "); + result.append(stackTrace); + } + + return result.toString(); + } + + public static Layout createLayout(String pattern, Charset charset) { + PatternLayout patternLayout = PatternLayout.newBuilder() + .withPattern(pattern) + .withCharset(charset) + .build(); + return new SingleLineExceptionLayout(patternLayout, charset); + } + } } diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 15bee4b009a22a..19ab65db6a9bb5 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -90,15 +90,6 @@ under the License. fe-common ${project.version} - - org.apache.logging.log4j - log4j-web - ${log4j2.version} - - - org.apache.logging.log4j - log4j-iostreams - org.springframework.boot spring-boot-devtools diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 5395de076ae223..e52ff5dea68c47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -104,9 +104,10 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" " + RUNTIME_LOG_PATTERN + "\n") - .append(" \n") + .append(" SingleLineExceptionLayout\n") + .append(" \n") .append(" \n") .append(" \n") .append(" \n") From 952e79ce167aa5a1c95752b608ab08d799bcba95 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Jun 2024 13:06:53 +0800 Subject: [PATCH 11/14] tmp1 --- bin/start_fe.sh | 5 +++-- .../main/java/org/apache/doris/common/LogUtils.java | 10 +++++++++- .../main/java/org/apache/doris/common/Log4jConfig.java | 9 ++++----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/bin/start_fe.sh b/bin/start_fe.sh index 64c348cb64e7ed..48f4b9b2d1cc4c 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -178,10 +178,11 @@ fi STDOUT_LOGGER="${LOG_DIR}/fe.out" log() { + cur_date=$(date +"%Y-%m-%d %H:%M:%S,$(date +%3N)") if [[ "${RUN_CONSOLE}" -eq 1 ]]; then - echo "$1" + echo "StdoutLogger ${cur_date} $1" else - echo "$1" >>"${STDOUT_LOGGER}" + echo "StdoutLogger ${cur_date} $1" >>"${STDOUT_LOGGER}" fi } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java index 1a18ec222cd40d..1194f305ed605c 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java @@ -21,6 +21,10 @@ import org.apache.logging.log4j.core.Layout; import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; import org.apache.logging.log4j.core.layout.AbstractStringLayout; import org.apache.logging.log4j.core.layout.PatternLayout; import org.apache.logging.log4j.core.util.StringBuilderWriter; @@ -50,6 +54,7 @@ public static void stderr(String message) { System.err.println(STDERR_LOG_MARKER + formattedTime() + " " + message); } + @Plugin(name = "NewCustLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true) public static class SingleLineExceptionLayout extends AbstractStringLayout { private final PatternLayout patternLayout; @@ -73,7 +78,10 @@ public String toSerializable(LogEvent event) { return result.toString(); } - public static Layout createLayout(String pattern, Charset charset) { + @PluginFactory + public static Layout createLayout( + @PluginAttribute(value = "pattern") String pattern, + @PluginAttribute(value = "charset", defaultString = "UTF-8") Charset charset) { PatternLayout patternLayout = PatternLayout.newBuilder() .withPattern(pattern) .withCharset(charset) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index e52ff5dea68c47..1d2596b7ccdd78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -94,9 +94,9 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append("\n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" " + RUNTIME_LOG_PATTERN + "\n") - .append(" \n") + .append(" \n") .append(" \n") .append(" \n") .append(" \n") @@ -104,10 +104,9 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" " + RUNTIME_LOG_PATTERN + "\n") - .append(" SingleLineExceptionLayout\n") - .append(" \n") + .append(" \n") .append(" \n") .append(" \n") .append(" \n") From eaa5bc4e4312359bb21cd4f1039daed970294eec Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Jun 2024 14:56:40 +0800 Subject: [PATCH 12/14] restore --- .../src/main/java/org/apache/doris/common/LogUtils.java | 7 ++++++- .../src/main/java/org/apache/doris/common/Log4jConfig.java | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java index 1194f305ed605c..554b1751f37666 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java @@ -54,7 +54,12 @@ public static void stderr(String message) { System.err.println(STDERR_LOG_MARKER + formattedTime() + " " + message); } - @Plugin(name = "NewCustLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true) + // TODO: this custom layout is not used in the codebase, but it is a good example of how to create a custom layout + // 1. Add log4j2.component.properties in fe/conf with content: + // log4j.layoutFactory=org.apache.doris.common.LogUtils$SingleLineExceptionLayout + // 2. Change PatternLayout in Log4jConfig.java to SingleLineExceptionLayout + @Plugin(name = "SingleLineExceptionLayout", category = Node.CATEGORY, + elementType = Layout.ELEMENT_TYPE, printObject = true) public static class SingleLineExceptionLayout extends AbstractStringLayout { private final PatternLayout patternLayout; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 1d2596b7ccdd78..5395de076ae223 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -94,9 +94,9 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append("\n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" " + RUNTIME_LOG_PATTERN + "\n") - .append(" \n") + .append(" \n") .append(" \n") .append(" \n") .append(" \n") From 32135f6d758bdd8651b77c5da037b3f0656d4b3e Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Jun 2024 15:12:46 +0800 Subject: [PATCH 13/14] format --- .../org/apache/doris/common/LogUtils.java | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java index 554b1751f37666..e13109a5e22f99 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java @@ -1,21 +1,19 @@ -/* - * // Licensed to the Apache Software Foundation (ASF) under one - * // or more contributor license agreements. See the NOTICE file - * // distributed with this work for additional information - * // regarding copyright ownership. The ASF licenses this file - * // to you under the Apache License, Version 2.0 (the - * // "License"); you may not use this file except in compliance - * // with the License. You may obtain a copy of the License at - * // - * // http://www.apache.org/licenses/LICENSE-2.0 - * // - * // Unless required by applicable law or agreed to in writing, - * // software distributed under the License is distributed on an - * // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * // KIND, either express or implied. See the License for the - * // specific language governing permissions and limitations - * // under the License. - */ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package org.apache.doris.common; From fc7bb66f4c5094c8674ad6afaa018292ec82f2be Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 3 Jun 2024 15:18:40 +0800 Subject: [PATCH 14/14] format --- bin/start_fe.sh | 1 + .../src/main/java/org/apache/doris/common/Config.java | 8 ++++---- .../src/main/java/org/apache/doris/common/LogUtils.java | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/start_fe.sh b/bin/start_fe.sh index 48f4b9b2d1cc4c..67a6925fea1901 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -178,6 +178,7 @@ fi STDOUT_LOGGER="${LOG_DIR}/fe.out" log() { + # same datetime format as in fe.log: 2024-06-03 14:54:41,478 cur_date=$(date +"%Y-%m-%d %H:%M:%S,$(date +%3N)") if [[ "${RUN_CONSOLE}" -eq 1 ]]; then echo "StdoutLogger ${cur_date} $1" diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index f165aec79e5756..63e45b3caa81ff 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -127,11 +127,11 @@ public class Config extends ConfigBase { @ConfField(description = {"是否压缩 FE 的 Audit 日志", "enable compression for FE audit log file"}) public static boolean audit_log_enable_compress = false; - @ConfField(description = {"是否使用文件记录日志。当使用 --console 启动 FE 时，全部日志会写入到标准输出。" - + "此时关闭关闭这个选项，不再使用文件记录日志。", + @ConfField(description = {"是否使用文件记录日志。当使用 --console 启动 FE 时，全部日志同时写入到标准输出和文件。" + + "如果关闭这个选项，不再使用文件记录日志。", "Whether to use file to record log. When starting FE with --console, " - + "all logs will be written to standard output. " - + "At this time, close this option and no longer use file to record log."}) + + "all logs will be written to both standard output and file. " + + "Close this option will no longer use file to record log."}) public static boolean enable_file_logger = true; @ConfField(mutable = false, masterOnly = false, diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java index e13109a5e22f99..2f93cdb8850b53 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java @@ -44,6 +44,8 @@ private static String formattedTime() { return dateTime.format(TIME_FORMATTER); } + // Developer should use `LogUtils.stdout` or `LogUtils.stderr` + // instead of `System.out` and `System.err`. public static void stdout(String message) { System.out.println(STDOUT_LOG_MARKER + formattedTime() + " " + message); }