diff --git a/bin/start_fe.sh b/bin/start_fe.sh index 9d51648ff4fd66..67a6925fea1901 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -176,6 +176,17 @@ if [[ ! -d "${LOG_DIR}" ]]; then mkdir -p "${LOG_DIR}" fi +STDOUT_LOGGER="${LOG_DIR}/fe.out" +log() { + # same datetime format as in fe.log: 2024-06-03 14:54:41,478 + cur_date=$(date +"%Y-%m-%d %H:%M:%S,$(date +%3N)") + if [[ "${RUN_CONSOLE}" -eq 1 ]]; then + echo "StdoutLogger ${cur_date} $1" + else + echo "StdoutLogger ${cur_date} $1" >>"${STDOUT_LOGGER}" + fi +} + # check java version and choose correct JAVA_OPTS java_version="$( set -e @@ -183,16 +194,16 @@ java_version="$( )" if [[ "${java_version}" -eq 17 ]]; then if [[ -z "${JAVA_OPTS_FOR_JDK_17}" ]]; then - echo "JAVA_OPTS_FOR_JDK_17 is not set in fe.conf" >>"${LOG_DIR}/fe.out" + log "JAVA_OPTS_FOR_JDK_17 is not set in fe.conf" exit 1 fi final_java_opt="${JAVA_OPTS_FOR_JDK_17}" else - echo "ERROR: The jdk_version is ${java_version}, must be 17." >>"${LOG_DIR}/fe.out" + log "ERROR: The jdk_version is ${java_version}, must be 17." exit 1 fi -echo "Using Java version ${java_version}" >>"${LOG_DIR}/fe.out" -echo "${final_java_opt}" >>"${LOG_DIR}/fe.out" +log "Using Java version ${java_version}" +log "${final_java_opt}" # add libs to CLASSPATH DORIS_FE_JAR= @@ -239,7 +250,8 @@ if [[ -n "${JACOCO_COVERAGE_OPT}" ]]; then coverage_opt="${JACOCO_COVERAGE_OPT}" fi -date >>"${LOG_DIR}/fe.out" +CUR_DATE=$(date) +log "start time: ${CUR_DATE}" if [[ "${HELPER}" != "" ]]; then # change it to '-helper' to be compatible with code in Frontend @@ -253,12 +265,12 @@ if [[ "${IMAGE_TOOL}" -eq 1 ]]; then echo "Internal error, USE IMAGE_TOOL like: ./start_fe.sh --image image_path" fi elif [[ "${RUN_DAEMON}" -eq 1 ]]; then - nohup ${LIMIT:+${LIMIT}} "${JAVA}" ${final_java_opt:+${final_java_opt}} -XX:-OmitStackTraceInFastThrow -XX:OnOutOfMemoryError="kill -9 %p" ${coverage_opt:+${coverage_opt}} org.apache.doris.DorisFE ${HELPER:+${HELPER}} "${METADATA_FAILURE_RECOVERY}" "$@" >>"${LOG_DIR}/fe.out" 2>&1 >"${STDOUT_LOGGER}" 2>&1 >"${LOG_DIR}/fe.out" 2>&1 >"${STDOUT_LOGGER}" 2>&1 com.amazonaws aws-java-sdk-s3 + + org.apache.logging.log4j + log4j-web + + + org.apache.logging.log4j + log4j-iostreams + + + org.apache.logging.log4j + log4j-core + doris-fe-common diff --git a/fe/fe-common/src/main/java/org/apache/doris/cloud/security/UrlSecurityChecker.java b/fe/fe-common/src/main/java/org/apache/doris/cloud/security/UrlSecurityChecker.java index 8d4056595e17e0..5f79343d0f9bd4 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/cloud/security/UrlSecurityChecker.java +++ b/fe/fe-common/src/main/java/org/apache/doris/cloud/security/UrlSecurityChecker.java @@ -18,6 +18,7 @@ package org.apache.doris.cloud.security; import org.apache.doris.common.Config; +import org.apache.doris.common.LogUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -38,8 +39,8 @@ public class UrlSecurityChecker extends SecurityChecker { urlSecurityStopCheckMethod = clazz.getMethod("stopSSRFNetHookChecking", String.class); } catch (Exception e) { LOG.warn("security_checker_class_name:{} exception:", Config.security_checker_class_name, e); + LogUtils.stderr("Failed to find com.aliyun.securitysdk.SecurityUtil's method"); e.printStackTrace(); - System.out.println("Failed to find com.aliyun.securitysdk.SecurityUtil's method"); } } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 8420e598d2b661..63e45b3caa81ff 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -127,6 +127,13 @@ public class Config extends ConfigBase { @ConfField(description = {"是否压缩 FE 的 Audit 日志", "enable compression for FE audit log file"}) public static boolean audit_log_enable_compress = false; + @ConfField(description = {"是否使用文件记录日志。当使用 --console 启动 FE 时，全部日志同时写入到标准输出和文件。" + + "如果关闭这个选项，不再使用文件记录日志。", + "Whether to use file to record log. When starting FE with --console, " + + "all logs will be written to both standard output and file. " + + "Close this option will no longer use file to record log."}) + public static boolean enable_file_logger = true; + @ConfField(mutable = false, masterOnly = false, description = {"是否检查table锁泄漏", "Whether to check table lock leaky"}) public static boolean check_table_lock_leaky = false; diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java new file mode 100644 index 00000000000000..2f93cdb8850b53 --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/common/LogUtils.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.layout.AbstractStringLayout; +import org.apache.logging.log4j.core.layout.PatternLayout; +import org.apache.logging.log4j.core.util.StringBuilderWriter; + +import java.io.PrintWriter; +import java.nio.charset.Charset; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +public class LogUtils { + + public static final String STDOUT_LOG_MARKER = "StdoutLogger "; + public static final String STDERR_LOG_MARKER = "StderrLogger "; + + private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss,SSS"); + + private static String formattedTime() { + LocalDateTime dateTime = LocalDateTime.now(); + return dateTime.format(TIME_FORMATTER); + } + + // Developer should use `LogUtils.stdout` or `LogUtils.stderr` + // instead of `System.out` and `System.err`. + public static void stdout(String message) { + System.out.println(STDOUT_LOG_MARKER + formattedTime() + " " + message); + } + + public static void stderr(String message) { + System.err.println(STDERR_LOG_MARKER + formattedTime() + " " + message); + } + + // TODO: this custom layout is not used in the codebase, but it is a good example of how to create a custom layout + // 1. Add log4j2.component.properties in fe/conf with content: + // log4j.layoutFactory=org.apache.doris.common.LogUtils$SingleLineExceptionLayout + // 2. Change PatternLayout in Log4jConfig.java to SingleLineExceptionLayout + @Plugin(name = "SingleLineExceptionLayout", category = Node.CATEGORY, + elementType = Layout.ELEMENT_TYPE, printObject = true) + public static class SingleLineExceptionLayout extends AbstractStringLayout { + + private final PatternLayout patternLayout; + + protected SingleLineExceptionLayout(PatternLayout patternLayout, Charset charset) { + super(charset); + this.patternLayout = patternLayout; + } + + @Override + public String toSerializable(LogEvent event) { + StringBuilder result = new StringBuilder(patternLayout.toSerializable(event)); + + if (event.getThrown() != null) { + StringBuilderWriter sw = new StringBuilderWriter(); + event.getThrown().printStackTrace(new PrintWriter(sw)); + String stackTrace = sw.toString().replace("\n", " ").replace("\r", " "); + result.append(stackTrace); + } + + return result.toString(); + } + + @PluginFactory + public static Layout createLayout( + @PluginAttribute(value = "pattern") String pattern, + @PluginAttribute(value = "charset", defaultString = "UTF-8") Charset charset) { + PatternLayout patternLayout = PatternLayout.newBuilder() + .withPattern(pattern) + .withCharset(charset) + .build(); + return new SingleLineExceptionLayout(patternLayout, charset); + } + } +} diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index ac697ae6c4e9f2..19ab65db6a9bb5 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -90,11 +90,6 @@ under the License. fe-common ${project.version} - - org.apache.logging.log4j - log4j-web - ${log4j2.version} - org.springframework.boot spring-boot-devtools diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java index 583f20687c69f4..3ca39bbfd0690e 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java @@ -24,6 +24,7 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.LdapConfig; import org.apache.doris.common.Log4jConfig; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.Version; import org.apache.doris.common.util.JdkUtils; @@ -386,11 +387,11 @@ private static CommandLineOptions parseArgs(String[] args) { } private static void printVersion() { - System.out.println("Build version: " + Version.DORIS_BUILD_VERSION); - System.out.println("Build time: " + Version.DORIS_BUILD_TIME); - System.out.println("Build info: " + Version.DORIS_BUILD_INFO); - System.out.println("Build hash: " + Version.DORIS_BUILD_HASH); - System.out.println("Java compile version: " + Version.DORIS_JAVA_COMPILE_VERSION); + LogUtils.stdout("Build version: " + Version.DORIS_BUILD_VERSION); + LogUtils.stdout("Build time: " + Version.DORIS_BUILD_TIME); + LogUtils.stdout("Build info: " + Version.DORIS_BUILD_INFO); + LogUtils.stdout("Build hash: " + Version.DORIS_BUILD_HASH); + LogUtils.stdout("Java compile version: " + Version.DORIS_JAVA_COMPILE_VERSION); LOG.info("Build version: {}", Version.DORIS_BUILD_VERSION); LOG.info("Build time: {}", Version.DORIS_BUILD_TIME); @@ -413,16 +414,16 @@ private static void checkCommandLineOptions(CommandLineOptions cmdLineOpts) { } else if (cmdLineOpts.runImageTool()) { File imageFile = new File(cmdLineOpts.getImagePath()); if (!imageFile.exists()) { - System.out.println("image does not exist: " + imageFile.getAbsolutePath() + LogUtils.stderr("image does not exist: " + imageFile.getAbsolutePath() + " . Please put an absolute path instead"); System.exit(-1); } else { - System.out.println("Start to load image: "); + LogUtils.stdout("Start to load image: "); try { MetaReader.read(imageFile, Env.getCurrentEnv()); - System.out.println("Load image success. Image file " + cmdLineOpts.getImagePath() + " is valid"); + LogUtils.stdout("Load image success. Image file " + cmdLineOpts.getImagePath() + " is valid"); } catch (Exception e) { - System.out.println("Load image failed. Image file " + cmdLineOpts.getImagePath() + " is invalid"); + LogUtils.stderr("Load image failed. Image file " + cmdLineOpts.getImagePath() + " is invalid"); LOG.warn("", e); } finally { System.exit(0); @@ -481,9 +482,7 @@ private static void tryLockProcess() { + "same time"); } - private static void releaseFileLockAndCloseFileChannel() { - if (processFileLock != null && processFileLock.isValid()) { try { processFileLock.release(); @@ -498,7 +497,6 @@ private static void releaseFileLockAndCloseFileChannel() { LOG.warn("release process lock file failed", ignored); } } - } public static void overwriteConfigs() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index d49e7dd73b27dc..a91ab2ea5ba434 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -103,6 +103,7 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.NereidsSqlCacheManager; import org.apache.doris.common.Pair; @@ -1508,7 +1509,7 @@ private void transferToMaster() { if (Config.enable_check_compatibility_mode) { String msg = "check metadata compatibility successfully"; LOG.info(msg); - System.out.println(msg); + LogUtils.stdout(msg); System.exit(0); } @@ -1612,7 +1613,7 @@ private void transferToMaster() { checkLowerCaseTableNames(); String msg = "master finished to replay journal, can write now."; - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); LOG.info(msg); // for master, there are some new thread pools need to register metric ThreadPoolManager.registerAllThreadPoolMetric(); @@ -2721,7 +2722,7 @@ public void notifyNewFETypeTransfer(FrontendNodeType newType) { try { String msg = "notify new FE type transfer: " + newType; LOG.warn(msg); - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); this.typeTransferQueue.put(newType); } catch (InterruptedException e) { LOG.error("failed to put new FE type: {}", newType, e); @@ -2739,7 +2740,7 @@ protected synchronized void runOneCycle() { newType = typeTransferQueue.take(); } catch (InterruptedException e) { LOG.error("got exception when take FE type from queue", e); - Util.stdoutWithTime("got exception when take FE type from queue. " + e.getMessage()); + LogUtils.stdout("got exception when take FE type from queue. " + e.getMessage()); System.exit(-1); } Preconditions.checkNotNull(newType); @@ -2821,7 +2822,7 @@ protected synchronized void runOneCycle() { // exit if master changed to any other type String msg = "transfer FE type from MASTER to " + newType.name() + ". exit"; LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); System.exit(-1); break; } @@ -6320,13 +6321,13 @@ public void checkReadyOrThrowTException() throws TException { private void replayJournalsAndExit() { replayJournal(-1); LOG.info("check metadata compatibility successfully"); - System.out.println("check metadata compatibility successfully"); + LogUtils.stdout("check metadata compatibility successfully"); if (Config.checkpoint_after_check_compatibility) { String imagePath = dumpImage(); String msg = "the new image file path is: " + imagePath; LOG.info(msg); - System.out.println(msg); + LogUtils.stdout(msg); } System.exit(0); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java index 8c840a876bb69a..5395de076ae223 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java @@ -21,15 +21,18 @@ import com.google.common.base.Strings; import com.google.common.collect.Maps; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.ConfigurationSource; import org.apache.logging.log4j.core.config.xml.XmlConfiguration; import org.apache.logging.log4j.core.lookup.Interpolator; import org.apache.logging.log4j.core.lookup.StrSubstitutor; +import org.apache.logging.log4j.io.IoBuilder; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.PrintStream; import java.nio.charset.StandardCharsets; import java.util.Map; @@ -54,20 +57,55 @@ private static void getXmlConfByStrategy(final String size, final String age) { } } + // Placeholders + private static final String RUNTIME_LOG_FORMAT_PLACEHOLDER = ""; + private static final String VERBOSE_MODULE_PLACEHOLDER = ""; + private static final String CONSOLE_APPENDER_PLACEHOLDER = ""; + private static final String RUNTIME_LOG_FILE_APPENDER_PLACEHOLDER = ""; + private static final String RUNTIME_LOG_WARN_FILE_APPENDER_PLACEHOLDER = ""; + private static final String AUDIT_CONSOLE_LOGGER_PLACEHOLDER = ""; + private static final String AUDIT_FILE_LOGGER_PLACEHOLDER = ""; + private static final String RUNTIME_LOG_MARKER_PLACEHOLDER = ""; + private static final String AUDIT_LOG_MARKER_PLACEHOLDER = ""; + + // Appender names + private static final String RUNTIME_LOG_CONSOLE_APPENDER = "Console"; + private static final String RUNTIME_LOG_FILE_APPENDER = "Sys"; + private static final String RUNTIME_LOG_WARN_FILE_APPENDER = "SysWF"; + private static final String AUDIT_LOG_CONSOLE_APPENDER = "AuditConsole"; + private static final String AUDIT_LOG_FILE_APPENDER = "AuditFile"; + + // Log patterns + private static final String RUNTIME_LOG_PATTERN + = RUNTIME_LOG_MARKER_PLACEHOLDER + "%d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)" + + RUNTIME_LOG_FORMAT_PLACEHOLDER + "%m%n"; + private static final String AUDIT_LOG_PATTERN + = AUDIT_LOG_MARKER_PLACEHOLDER + "%d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n"; + + // Log markers + private static final String RUNTIME_LOG_MARKER = "RuntimeLogger "; + private static final String AUDIT_LOG_MARKER = "AuditLogger "; + + // @formatter:off static { // CHECKSTYLE OFF xmlConfTemplateBuilder.append("\n") .append("\n\n") .append("\n") .append(" \n") - .append(" ") + .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n\n") + .append(" " + RUNTIME_LOG_PATTERN + "\n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n\n") + .append(" " + AUDIT_LOG_PATTERN + "\n") + .append(" \n") + .append(" \n") + .append(" \n") + .append(" \n") + .append(" " + RUNTIME_LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") @@ -83,9 +121,9 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} %p (%t|%tid)%m%n\n") + .append(" " + RUNTIME_LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") @@ -101,9 +139,9 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") + .append(" \n") .append(" \n") - .append(" %d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n\n") + .append(" " + AUDIT_LOG_PATTERN + "\n") .append(" \n") .append(" \n") .append(" \n") @@ -122,18 +160,20 @@ private static void getXmlConfByStrategy(final String size, final String age) { .append(" \n") .append(" \n") .append(" \n") - .append(" \n") - .append(" \n") - .append(" \n") + .append(" " + RUNTIME_LOG_FILE_APPENDER_PLACEHOLDER + "\n") + .append(" " + RUNTIME_LOG_WARN_FILE_APPENDER_PLACEHOLDER + "\n") + .append(" " + CONSOLE_APPENDER_PLACEHOLDER + "\n") .append(" \n") .append(" \n") - .append(" \n") + .append(" " + AUDIT_FILE_LOGGER_PLACEHOLDER + "\n") + .append(" " + AUDIT_CONSOLE_LOGGER_PLACEHOLDER + "\n") .append(" \n") - .append(" \n") + .append(" " + VERBOSE_MODULE_PLACEHOLDER + "\n") .append(" \n") .append(""); // CHECKSTYLE ON } + // @formatter:on private static StrSubstitutor strSub; private static String sysLogLevel; @@ -210,24 +250,33 @@ private static void reconfig() throws IOException { for (String s : auditModules) { sb.append(""); } - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", - sb.toString()); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(VERBOSE_MODULE_PLACEHOLDER, sb.toString()); if (sysLogMode.equalsIgnoreCase("NORMAL")) { - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", - " [%C{1}.%M():%L] "); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_FORMAT_PLACEHOLDER, " [%C{1}.%M():%L] "); } else { - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", " "); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_FORMAT_PLACEHOLDER, " "); if (sysLogMode.equalsIgnoreCase("ASYNC")) { newXmlConfTemplate = newXmlConfTemplate.replaceAll("Root", "AsyncRoot"); } } + if (Config.enable_file_logger) { + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_FILE_APPENDER_PLACEHOLDER, + "\n"); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_WARN_FILE_APPENDER_PLACEHOLDER, + "\n"); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(AUDIT_FILE_LOGGER_PLACEHOLDER, + "\n"); + } + if (foreground) { - StringBuilder consoleLogger = new StringBuilder(); - consoleLogger.append("\n"); - newXmlConfTemplate = newXmlConfTemplate.replaceAll("", - consoleLogger.toString()); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(RUNTIME_LOG_MARKER_PLACEHOLDER, RUNTIME_LOG_MARKER); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(AUDIT_LOG_MARKER_PLACEHOLDER, AUDIT_LOG_MARKER); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(CONSOLE_APPENDER_PLACEHOLDER, + "\n"); + newXmlConfTemplate = newXmlConfTemplate.replaceAll(AUDIT_CONSOLE_LOGGER_PLACEHOLDER, + "\n"); } Map properties = Maps.newHashMap(); @@ -256,9 +305,7 @@ private static void reconfig() throws IOException { strSub = new StrSubstitutor(new Interpolator(properties)); newXmlConfTemplate = strSub.replace(newXmlConfTemplate); - System.out.println("====="); - System.out.println(newXmlConfTemplate); - System.out.println("====="); + LogUtils.stdout("=====\n" + newXmlConfTemplate + "\n====="); logXmlConfTemplate = newXmlConfTemplate; SpringLog4j2Config.writeSpringLogConf(customConfDir); @@ -277,6 +324,17 @@ private static void reconfig() throws IOException { } catch (Exception e) { throw new IOException("Error occurred while configuring Log4j", e); } + + redirectStd(); + } + + private static void redirectStd() { + PrintStream logPrintStream = IoBuilder.forLogger(LogManager.getLogger("system.out")).setLevel(Level.INFO) + .buildPrintStream(); + System.setOut(logPrintStream); + PrintStream errorPrintStream = IoBuilder.forLogger(LogManager.getLogger("system.err")).setLevel(Level.ERROR) + .buildPrintStream(); + System.setErr(errorPrintStream); } public static String getLogXmlConfTemplate() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java index 0a207925fde846..c0a544d825dcce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DocGenerator.java @@ -19,6 +19,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ConfigBase.ConfField; +import org.apache.doris.common.LogUtils; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.VariableMgr; @@ -148,7 +149,7 @@ private String genFEConfigDoc(Lang lang) throws IllegalAccessException { sortedDoc.put(field.getName(), res); } } catch (Exception e) { - System.out.println("Failed to generate doc for field: " + field.getName()); + LogUtils.stderr("Failed to generate doc for field: " + field.getName()); throw e; } } @@ -204,7 +205,7 @@ private String genSessionVariableDoc(Lang lang) throws IllegalAccessException { sortedDoc.put(field.getAnnotation(VariableMgr.VarAttr.class).name(), res); } } catch (Exception e) { - System.out.println("Failed to generate doc for " + field.getName()); + LogUtils.stderr("Failed to generate doc for " + field.getName()); throw e; } } @@ -217,7 +218,7 @@ private String genSessionVariableDoc(Lang lang) throws IllegalAccessException { sortedDoc.put(field.getAnnotation(VariableMgr.VarAttr.class).name(), res); } } catch (Exception e) { - System.out.println("Failed to generate doc for field: " + field.getName()); + LogUtils.stderr("Failed to generate doc for field: " + field.getName()); throw e; } } @@ -288,7 +289,7 @@ public static void main(String[] args) { sessionVariableDocOutputPath, sessionVariableDocOutputPathCN); try { docGenerator.generate(); - System.out.println("Done!"); + LogUtils.stdout("Done!"); } catch (Exception e) { log.info("failed to generate doc", e); System.exit(-1); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java index 34a604eb0d3b76..7d3a0402bcbbb3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/JdkUtils.java @@ -18,6 +18,7 @@ package org.apache.doris.common.util; import org.apache.doris.common.Config; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.Version; import com.google.common.annotations.VisibleForTesting; @@ -42,7 +43,7 @@ public static boolean checkJavaVersion() { int runtimeVersion = JdkUtils.getJavaVersionAsInteger(javaRuntimeVersionStr); if (runtimeVersion < compileVersion) { - System.out.println("The runtime java version " + javaRuntimeVersionStr + " is less than " + LogUtils.stdout("The runtime java version " + javaRuntimeVersionStr + " is less than " + "compile version " + javaCompileVersionStr); return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PlatformName.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PlatformName.java deleted file mode 100644 index 0953c37196d2ee..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PlatformName.java +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.common.util; - -public class PlatformName { - - private static final String platformName = System.getProperty("os.name") + "-" - + System.getProperty("os.arch") + "-" - + System.getProperty("sun.arch.data.model"); - - public static String getPlatformName() { - return platformName; - } - - public static void main(String[] args) { - System.out.println(platformName); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index 4e4c6a947359fb..e73df139e01a8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -424,10 +424,6 @@ public static boolean getBooleanPropertyOrDefault(String valStr, boolean default } } - public static void stdoutWithTime(String msg) { - System.out.println("[" + TimeUtils.longToTimeString(System.currentTimeMillis()) + "] " + msg); - } - // not support encode negative value now public static void encodeVarint64(long source, DataOutput out) throws IOException { assert source >= 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java index 5ba686fc6882b3..46e8d1db47c723 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java @@ -149,7 +149,6 @@ public Optional initSchema() { TableInfo tableInfo = ((LakeSoulExternalCatalog) catalog).getLakeSoulTable(dbName, name); String tableSchema = tableInfo.getTableSchema(); DBUtil.TablePartitionKeys partitionKeys = DBUtil.parseTableInfoPartitions(tableInfo.getPartitions()); - System.out.println(tableSchema); Schema schema; try { schema = Schema.fromJSON(tableSchema); diff --git a/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java b/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java index 2ff66a5aad0162..302717c6050e8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java +++ b/fe/fe-core/src/main/java/org/apache/doris/ha/BDBStateChangeListener.java @@ -18,7 +18,7 @@ package org.apache.doris.ha; import org.apache.doris.catalog.Env; -import org.apache.doris.common.util.Util; +import org.apache.doris.common.LogUtils; import com.google.common.base.Preconditions; import com.sleepycat.je.rep.StateChangeEvent; @@ -57,7 +57,7 @@ public synchronized void stateChange(StateChangeEvent sce) throws RuntimeExcepti default: { String msg = "this node is " + sce.getState().name(); LOG.warn(msg); - Util.stdoutWithTime(msg); + LogUtils.stdout(msg); return; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java index 47a41b8226ad01..d2784a05e70c7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBEnvironment.java @@ -105,8 +105,7 @@ public void setup(File envHome, String selfNodeName, String selfNodeHostPort, DbResetRepGroup resetUtility = new DbResetRepGroup( envHome, PALO_JOURNAL_GROUP, selfNodeName, selfNodeHostPort); resetUtility.reset(); - LOG.warn("WARNING: metadata recovery mode, group has been reset."); - System.out.println("WARNING: metadata recovery mode, group has been reset."); + LOG.warn("metadata recovery mode, group has been reset."); } // set replication config diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java index 0d342a986cd40f..9e78b5eb47a14b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java @@ -20,10 +20,10 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.LogUtils; import org.apache.doris.common.io.DataOutputBuffer; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.NetUtils; -import org.apache.doris.common.util.Util; import org.apache.doris.journal.Journal; import org.apache.doris.journal.JournalBatch; import org.apache.doris.journal.JournalCursor; @@ -120,7 +120,7 @@ public synchronized void rollJournal() { + "journal id: %d, current db: %s, expected db count: %d", newName, currentDbName, newNameVerify); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } } @@ -193,7 +193,7 @@ public synchronized long write(JournalBatch batch) throws IOException { String msg = "write bdb failed. will exit. the first journalId: " + firstId + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } catch (DatabaseException e) { LOG.error("catch an exception when writing to database. sleep and retry. the first journal id {}", @@ -218,7 +218,7 @@ public synchronized long write(JournalBatch batch) throws IOException { String msg = "write bdb failed. will exit. the first journalId: " + firstId + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); return 0; // unreachable! } @@ -285,7 +285,7 @@ public synchronized long write(short op, Writable writable) throws IOException { String msg = "write bdb failed. will exit. journalId: " + id + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } catch (DatabaseException e) { LOG.error("catch an exception when writing to database. sleep and retry. journal id {}", id, e); @@ -301,7 +301,7 @@ public synchronized long write(short op, Writable writable) throws IOException { String msg = "write bdb failed. will exit. journalId: " + id + ", bdb database Name: " + currentJournalDB.getDatabaseName(); LOG.error(msg); - Util.stdoutWithTime(msg); + LogUtils.stderr(msg); System.exit(-1); } return id; @@ -357,7 +357,7 @@ public JournalEntity read(long journalId) { LOG.warn("", e); } } else { - System.out.println("No record found for key '" + journalId + "'."); + LOG.warn("No record found for key '{}'.", journalId); } } catch (Exception e) { LOG.warn("catch an exception when get JournalEntity. key:{}", journalId, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java index 5466df71ddcc8a..8c48cab42c5df6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJournalCursor.java @@ -46,7 +46,7 @@ public class BDBJournalCursor implements JournalCursor { public static BDBJournalCursor getJournalCursor(BDBEnvironment env, long fromKey, long toKey) { if (toKey < fromKey || fromKey < 0) { - System.out.println("Invalid key range!"); + LOG.warn("Invalid key range! fromKey:{} toKey:{}", fromKey, toKey); return null; } BDBJournalCursor cursor = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java index cc9defa378bf57..55c15475282c36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBTool.java @@ -18,6 +18,7 @@ package org.apache.doris.journal.bdbje; import org.apache.doris.catalog.Env; +import org.apache.doris.common.LogUtils; import org.apache.doris.journal.JournalEntity; import org.apache.doris.meta.MetaContext; @@ -66,7 +67,7 @@ public boolean run() { env = new Environment(new File(metaPath), envConfig); } catch (DatabaseException e) { LOG.warn("", e); - System.err.println("Failed to open BDBJE env: " + Env.getCurrentEnv().getBdbDir() + ". exit"); + LogUtils.stderr("Failed to open BDBJE env: " + Env.getCurrentEnv().getBdbDir() + ". exit"); return false; } Preconditions.checkNotNull(env); @@ -75,7 +76,7 @@ public boolean run() { if (options.isListDbs()) { // list all databases List dbNames = env.getDatabaseNames(); - System.out.println(JSONArray.toJSONString(dbNames)); + LogUtils.stdout(JSONArray.toJSONString(dbNames)); return true; } else { // db operations @@ -90,7 +91,7 @@ public boolean run() { // get db stat Map statMap = Maps.newHashMap(); statMap.put("count", String.valueOf(db.count())); - System.out.println(JSONObject.toJSONString(statMap)); + LogUtils.stdout(JSONObject.toJSONString(statMap)); return true; } else { // set from key @@ -99,7 +100,7 @@ public boolean run() { try { fromKey = Long.valueOf(fromKeyStr); } catch (NumberFormatException e) { - System.err.println("Not a valid from key: " + fromKeyStr); + LogUtils.stderr("Not a valid from key: " + fromKeyStr); return false; } @@ -109,13 +110,13 @@ public boolean run() { try { endKey = Long.valueOf(options.getEndKey()); } catch (NumberFormatException e) { - System.err.println("Not a valid end key: " + options.getEndKey()); + LogUtils.stderr("Not a valid end key: " + options.getEndKey()); return false; } } if (fromKey > endKey) { - System.err.println("from key should less than or equal to end key[" + LogUtils.stderr("from key should less than or equal to end key[" + fromKey + " vs. " + endKey + "]"); return false; } @@ -132,7 +133,7 @@ public boolean run() { } } catch (Exception e) { LOG.warn("", e); - System.err.println("Failed to run bdb tools"); + LogUtils.stderr("Failed to run bdb tools"); return false; } return true; @@ -155,15 +156,15 @@ private void getValueByKey(Database db, Long key) entity.readFields(in); } catch (Exception e) { LOG.warn("", e); - System.err.println("Fail to read journal entity for key: " + key + ". reason: " + e.getMessage()); + LogUtils.stderr("Fail to read journal entity for key: " + key + ". reason: " + e.getMessage()); System.exit(-1); } - System.out.println("key: " + key); - System.out.println("op code: " + entity.getOpCode()); - System.out.println("value: " + entity.getData().toString()); + LogUtils.stdout("key: " + key); + LogUtils.stdout("op code: " + entity.getOpCode()); + LogUtils.stdout("value: " + entity.getData().toString()); } else if (status == OperationStatus.NOTFOUND) { - System.out.println("key: " + key); - System.out.println("value: NOT FOUND"); + LogUtils.stdout("key: " + key); + LogUtils.stdout("value: NOT FOUND"); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java index 45ae377a40171c..8e39f8a6a616be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournal.java @@ -84,7 +84,7 @@ public synchronized void rollJournal() { try { storage = new Storage(imageDir); if (journalId.get() == storage.getEditsSeq()) { - System.out.println("Does not need to roll!"); + LOG.warn("Does not need to roll! journalId: {}, editsSeq: {}", journalId.get(), storage.getEditsSeq()); return; } if (outputStream != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java index d570719dd3a13d..d24e394927b1cf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/local/LocalJournalCursor.java @@ -50,7 +50,7 @@ public final class LocalJournalCursor implements JournalCursor { public static LocalJournalCursor getJournalCursor(String imageDir, long fromKey, long toKey) { if (toKey < fromKey && toKey != -1 || fromKey < 0) { - System.out.println("Invalid key range!"); + LOG.warn("Invalid fromKey:{} toKey:{}", fromKey, toKey); return null; } long newToKey = toKey; @@ -95,7 +95,7 @@ private LocalJournalCursor(String imageDir, long fromKey, long toKey) throws IOE } if (fileName == null) { - System.out.println("Can not find the key:" + fromKey); + LOG.warn("Can not find the key:{}", fromKey); throw new IOException(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java index 0e58f1bc976ba5..ae3aa52c4f2094 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java @@ -79,9 +79,4 @@ public void execute(JobContext context) { public boolean isOnce() { return false; } - - private void printTraceLog(RuleType ruleType, String traceBefore, String traceAfter) { - System.out.println("========== " + getClass().getSimpleName() + " " + ruleType - + " ==========\nbefore:\n" + traceBefore + "\n\nafter:\n" + traceAfter + "\n"); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index d440cf77cac5e7..d5177844868373 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -446,8 +446,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { Frontend fe = (Frontend) journal.getData(); env.replayDropFrontend(fe); if (fe.getNodeName().equals(Env.getCurrentEnv().getNodeName())) { - System.out.println("current fe " + fe + " is removed. will exit"); - LOG.info("current fe " + fe + " is removed. will exit"); + LOG.warn("current fe {} is removed. will exit", fe); System.exit(-1); } break; diff --git a/fe/pom.xml b/fe/pom.xml index 2eaa31c43cd814..c443a07b001dc1 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -824,6 +824,11 @@ under the License. log4j-core ${log4j2.version} + + org.apache.logging.log4j + log4j-iostreams + ${log4j2.version} + org.apache.logging.log4j log4j-slf4j-impl