From d0aa2e050f24d1f74e0dea0ca9bdd5de293fcc7b Mon Sep 17 00:00:00 2001
From: w41ter <w41ter.l@gmail.com>
Date: Fri, 16 Aug 2024 08:57:02 +0000
Subject: [PATCH] [opt](log) refine the cloud logger

---
 cloud/CMakeLists.txt         |  3 +++
 cloud/script/start.sh        |  6 +++++
 cloud/src/common/config.h    |  5 ++++
 cloud/src/common/logging.cpp | 47 ++++++++++++++++++++++++++++++++----
 4 files changed, 56 insertions(+), 5 deletions(-)

diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt
index bc3f56642821b0..bb697d791e7116 100644
--- a/cloud/CMakeLists.txt
+++ b/cloud/CMakeLists.txt
@@ -457,6 +457,9 @@ if (NOT EXISTS ${THIRDPARTY_DIR}/include/foundationdb)
     execute_process(COMMAND "tar" "xf" "${THIRDPARTY_SRC}/${FDB_LIB}" "-C" "${THIRDPARTY_DIR}/")
 endif ()
 
+# enable glog custom prefix
+add_definitions(-DGLOG_CUSTOM_PREFIX_SUPPORT)
+
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lfdb_c -L${THIRDPARTY_DIR}/lib")
 
 add_subdirectory(${SRC_DIR}/common)
diff --git a/cloud/script/start.sh b/cloud/script/start.sh
index ebe2f88e16c2b6..5a61d3534b565d 100644
--- a/cloud/script/start.sh
+++ b/cloud/script/start.sh
@@ -34,12 +34,14 @@ fi
 
 RUN_DAEMON=0
 RUN_VERSION=0
+RUN_CONSOLE=0
 for arg; do
     shift
     [[ "${arg}" = "--daemonized" ]] && RUN_DAEMON=1 && continue
     [[ "${arg}" = "-daemonized" ]] && RUN_DAEMON=1 && continue
     [[ "${arg}" = "--daemon" ]] && RUN_DAEMON=1 && continue
     [[ "${arg}" = "--version" ]] && RUN_VERSION=1 && continue
+    [[ "${arg}" = "--console" ]] && RUN_CONSOLE=1 && continue
     set -- "$@" "${arg}"
 done
 # echo "$@" "daemonized=${daemonized}"}
@@ -137,6 +139,10 @@ if [[ "${RUN_DAEMON}" -eq 1 ]]; then
     tail -n10 "${DORIS_HOME}/log/${process}.out" | grep 'working directory' -B1 -A10
     echo "please check process log for more details"
     echo ""
+elif [[ "${RUN_CONSOLE}" -eq 1 ]]; then
+    export DORIS_LOG_TO_STDERR=1
+    date
+    "${bin}" "$@" 2>&1
 else
     "${bin}" "$@"
 fi
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index cb4bee9648e254..2b00e3d2245523 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -47,6 +47,11 @@ CONF_Int32(warn_log_filenum_quota, "1");
 CONF_Bool(log_immediate_flush, "false");
 CONF_Strings(log_verbose_modules, ""); // Comma seprated list: a.*,b.*
 CONF_Int32(log_verbose_level, "5");
+// Whether to use file to record log. When starting Cloud with --console,
+// all logs will be written to both standard output and file.
+// Disable this option will no longer use file to record log.
+// Only works when starting Cloud with --console.
+CONF_Bool(enable_file_logger, "true");
 
 // recycler config
 CONF_mInt64(recycle_interval_seconds, "3600");
diff --git a/cloud/src/common/logging.cpp b/cloud/src/common/logging.cpp
index 838e8892633307..65f9048a4df58d 100644
--- a/cloud/src/common/logging.cpp
+++ b/cloud/src/common/logging.cpp
@@ -22,6 +22,7 @@
 #include <glog/logging.h>
 #include <glog/vlog_is_on.h>
 
+#include <iomanip>
 #include <iostream>
 #include <mutex>
 
@@ -72,6 +73,28 @@ void AnnotateTag::format_tag_list(std::ostream& stream) {
     }
 }
 
+void custom_prefix(std::ostream& s, const google::LogMessageInfo& l, void*) {
+    // Add prefix "RuntimeLogger ".
+    s << "RuntimeLogger ";
+    // Same as in fe.log
+    // The following is same as default log format. eg:
+    // I20240605 15:25:15.677153 1763151 meta_service_txn.cpp:481] msg...
+    s << l.severity[0];
+    s << std::setw(4) << 1900 + l.time.year();
+    s << std::setw(2) << 1 + l.time.month();
+    s << std::setw(2) << l.time.day();
+    s << ' ';
+    s << std::setw(2) << l.time.hour() << ':';
+    s << std::setw(2) << l.time.min() << ':';
+    s << std::setw(2) << l.time.sec() << ".";
+    s << std::setw(6) << l.time.usec();
+    s << ' ';
+    s << std::setfill(' ') << std::setw(5);
+    s << l.thread_id << std::setfill('0');
+    s << ' ';
+    s << l.filename << ':' << l.line_number << "]";
+}
+
 /**
  * @param basename the basename of log file
  * @return true for success
@@ -82,10 +105,19 @@ bool init_glog(const char* basename) {
     std::lock_guard<std::mutex> logging_lock(mtx);
     if (inited) return true;
 
-    FLAGS_alsologtostderr = false;
-    // Don't log to stderr except fatal level
-    // so fatal log can output to be.out .
-    FLAGS_stderrthreshold = google::ERROR;
+    bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr);
+    if (log_to_console) {
+        if (config::enable_file_logger) {
+            FLAGS_alsologtostderr = true;
+        } else {
+            FLAGS_logtostderr = true;
+        }
+    } else {
+        FLAGS_alsologtostderr = false;
+        // Don't log to stderr except fatal level
+        // so fatal log can output to be.out .
+        FLAGS_stderrthreshold = google::ERROR;
+    }
 
     // Set glog log dir
     FLAGS_log_dir = config::log_dir;
@@ -128,7 +160,12 @@ bool init_glog(const char* basename) {
         if (i.empty()) continue;
         google::SetVLOGLevel(i.c_str(), config::log_verbose_level);
     }
-    google::InitGoogleLogging(basename);
+    if (log_to_console) {
+        // Only add prefix if log output to stderr
+        google::InitGoogleLogging(basename, &custom_prefix);
+    } else {
+        google::InitGoogleLogging(basename);
+    }
     inited = true;
     return true;
 }