[Enhancement] Add more comprehensive prometheus jvm thread metrics on fe

docs/en/administrator-guide/operation/monitor-metrics/fe-metrics.md

@@ -75,3 +75,51 @@ Value of the `Tcp: OutSegs` field in `/proc/net/snmp`. Represents the number of
 Use `(NEW_tcp_retrans_segs - OLD_tcp_retrans_segs) / (NEW_tcp_out_segs - OLD_tcp_out_segs)` can calculate the retrans rate of TCP packets.
 
 Usually used to troubleshoot network problems.
+
+### `jvm_thread{type="count"}`
+
+Value of the `count` type in `jvm_thread`. Represents the current number of live threads including both daemon and non-daemon threads.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="peak_count"}`
+
+Value of the `peak_count` type in `jvm_thread`. Represents the current number of live threads including both daemon and non-daemon threads.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="new_count"}`
+
+Value of the `new_count` type in `jvm_thread`. Represents the current number of threads which state is NEW.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="runnable_count"}`
+
+Value of the `runnable_count` type in `jvm_thread`. Represents the current number of threads which state is RUNNABLE.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="blocked_count"}`
+
+Value of the `blocked_count` type in `jvm_thread`. Represents the current number of threads which state is BLOCKED.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="waiting_count"}`
+
+Value of the `waiting_count` type in `jvm_thread`. Represents the current number of threads which state is WAITING.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="timed_waiting_count"}`
+
+Value of the `timed_waiting_count` type in `jvm_thread`. Represents the current number of threads which state is TIMED_WAITING.
+
+Usually used to troubleshoot jvm threads problems for FE.
+
+### `jvm_thread{type="terminated_count"}`
+
+Value of the `terminated_count` type in `jvm_thread`. Represents the current number of threads which state is TERMINATED.
+
+Usually used to troubleshoot jvm threads problems for FE.

docs/zh-CN/administrator-guide/operation/monitor-metrics/fe-metrics.md

@@ -75,3 +75,51 @@ FE 的监控项可以通过以下方式访问：
 通过 `(NEW_tcp_tcp_retrans_segs - OLD_tcp_retrans_segs) / (NEW_tcp_out_segs - OLD_tcp_out_segs)` 可以计算 TCP 重传率。
 
 通常用于排查网络问题。
+
+### `jvm_thread{type="count"}`
+
+该监控项表示FE节点当前JVM总的线程数量，包含daemon线程和非daemon线程。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="peak_count"}`
+
+该监控项表示FE节点从JVM启动以来的最大峰值线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="new_count"}`
+
+该监控项表示FE节点JVM中处于NEW状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="runnable_count"}`
+
+该监控项表示FE节点JVM中处于RUNNABLE状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="blocked_count"}`
+
+该监控项表示FE节点JVM中处于BLOCKED状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="waiting_count"}`
+
+该监控项表示FE节点JVM中处于WAITING状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="timed_waiting_count"}`
+
+该监控项表示FE节点JVM中处于TIMED_WAITING状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。
+
+### `jvm_thread{type="terminated_count"}`
+
+该监控项表示FE节点JVM中处于TERMINATED状态的线程数量。
+
+通常用于排查FE节点的JVM线程运行问题。

fe/fe-core/src/main/java/org/apache/doris/common/proc/JvmProcDir.java

@@ -94,6 +94,12 @@ public ProcResult fetchResult() throws AnalysisException {
         Threads threads = jvmStats.getThreads();
         result.addRow(genRow("threads count", threads.getCount()));
         result.addRow(genRow("threads peak count", threads.getPeakCount()));
+        result.addRow(genRow("threads new count", threads.getThreadsNewCount()));
+        result.addRow(genRow("threads runnable count", threads.getThreadsRunnableCount()));
+        result.addRow(genRow("threads blocked count", threads.getThreadsBlockedCount()));
+        result.addRow(genRow("threads waiting count", threads.getThreadsWaitingCount()));
+        result.addRow(genRow("threads timed_waiting count", threads.getThreadsTimedWaitingCount()));
+        result.addRow(genRow("threads terminated count", threads.getThreadsTerminatedCount()));
 
         return result;
     }

fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java

@@ -134,6 +134,12 @@ public void visitJvm(StringBuilder sb, JvmStats jvmStats) {
         sb.append(Joiner.on(" ").join(TYPE, JVM_THREAD, "gauge\n"));
         sb.append(JVM_THREAD).append("{type=\"count\"} ").append(threads.getCount()).append("\n");
         sb.append(JVM_THREAD).append("{type=\"peak_count\"} ").append(threads.getPeakCount()).append("\n");
+        sb.append(JVM_THREAD).append("{type=\"new_count\"} ").append(threads.getThreadsNewCount()).append("\n");
+        sb.append(JVM_THREAD).append("{type=\"runnable_count\"} ").append(threads.getThreadsRunnableCount()).append("\n");
+        sb.append(JVM_THREAD).append("{type=\"blocked_count\"} ").append(threads.getThreadsBlockedCount()).append("\n");
+        sb.append(JVM_THREAD).append("{type=\"waiting_count\"} ").append(threads.getThreadsWaitingCount()).append("\n");
+        sb.append(JVM_THREAD).append("{type=\"timed_waiting_count\"} ").append(threads.getThreadsTimedWaitingCount()).append("\n");
+        sb.append(JVM_THREAD).append("{type=\"terminated_count\"} ").append(threads.getThreadsTerminatedCount()).append("\n");
         return;
     }
 

fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/JvmStats.java

@@ -29,6 +29,7 @@
 import java.lang.management.MemoryUsage;
 import java.lang.management.RuntimeMXBean;
 import java.lang.management.ThreadMXBean;
+import java.lang.management.ThreadInfo;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -82,7 +83,28 @@ public static JvmStats jvmStats() {
         }
         Mem mem = new Mem(heapCommitted, heapUsed, heapMax, nonHeapCommitted, nonHeapUsed,
                 Collections.unmodifiableList(pools));
-        Threads threads = new Threads(threadMXBean.getThreadCount(), threadMXBean.getPeakThreadCount());
+
+        int threadsNew = 0;
+        int threadsRunnable = 0;
+        int threadsBlocked = 0;
+        int threadsWaiting = 0;
+        int threadsTimedWaiting = 0;
+        int threadsTerminated = 0;
+        long threadIds[] = threadMXBean.getAllThreadIds();
+        for (ThreadInfo threadInfo : threadMXBean.getThreadInfo(threadIds, 0)) {
+            if (threadInfo == null) continue; // race protection
+            switch (threadInfo.getThreadState()) {
+                case NEW:           threadsNew++;           break;
+                case RUNNABLE:      threadsRunnable++;      break;
+                case BLOCKED:       threadsBlocked++;       break;
+                case WAITING:       threadsWaiting++;       break;
+                case TIMED_WAITING: threadsTimedWaiting++;  break;
+                case TERMINATED:    threadsTerminated++;    break;
+                default:                                    break;
+            }
+        }
+        Threads threads = new Threads(threadMXBean.getThreadCount(), threadMXBean.getPeakThreadCount(), threadsNew,
+                threadsRunnable, threadsBlocked, threadsWaiting, threadsTimedWaiting, threadsTerminated);
 
         List<GarbageCollectorMXBean> gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans();
         GarbageCollector[] collectors = new GarbageCollector[gcMxBeans.size()];
@@ -192,6 +214,12 @@ static final class Fields {
         static final String THREADS = "threads";
         static final String COUNT = "count";
         static final String PEAK_COUNT = "peak_count";
+        static final String NEW_COUNT = "new_count";
+        static final String RUNNING_COUNT = "runnable_count";
+        static final String BLOCKED_COUNT = "blocked_count";
+        static final String WAITING_COUNT = "waiting_count";
+        static final String TIMED_WAITING_COUNT = "timed_waiting_count";
+        static final String TERMINATED_COUNT = "terminated_count";
 
         static final String GC = "gc";
         static final String COLLECTORS = "collectors";
@@ -264,10 +292,23 @@ public static class Threads {
 
         private final int count;
         private final int peakCount;
-
-        public Threads(int count, int peakCount) {
+        private final int threadsNewCount;
+        private final int threadsRunnableCount;
+        private final int threadsBlockedCount;
+        private final int threadsWaitingCount;
+        private final int threadsTimedWaitingCount;
+        private final int threadsTerminatedCount;
+
+        public Threads(int count, int peakCount, int threadsNewCount, int threadsRunnableCount, int threadsBlockedCount,
+                       int threadsWaitingCount, int threadsTimedWaitingCount, int threadsTerminatedCount) {
             this.count = count;
             this.peakCount = peakCount;
+            this.threadsNewCount = threadsNewCount;
+            this.threadsRunnableCount = threadsRunnableCount;
+            this.threadsBlockedCount = threadsBlockedCount;
+            this.threadsWaitingCount = threadsWaitingCount;
+            this.threadsTimedWaitingCount = threadsTimedWaitingCount;
+            this.threadsTerminatedCount = threadsTerminatedCount;
         }
 
         public int getCount() {
@@ -278,6 +319,30 @@ public int getPeakCount() {
             return peakCount;
         }
 
+        public int getThreadsNewCount() {
+            return threadsNewCount;
+        }
+
+        public int getThreadsRunnableCount() {
+            return threadsRunnableCount;
+        }
+
+        public int getThreadsBlockedCount() {
+            return threadsBlockedCount;
+        }
+
+        public int getThreadsWaitingCount() {
+            return threadsWaitingCount;
+        }
+
+        public int getThreadsTimedWaitingCount() {
+            return threadsTimedWaitingCount;
+        }
+
+        public int getThreadsTerminatedCount() {
+            return threadsTerminatedCount;
+        }
+
         @Override
         public String toString() {
             StringBuilder sb = new StringBuilder();