diff --git a/docs/en/administrator-guide/operation/monitor-metrics/fe-metrics.md b/docs/en/administrator-guide/operation/monitor-metrics/fe-metrics.md index 7a2afae78e4796..4c4e3f7dfd1735 100644 --- a/docs/en/administrator-guide/operation/monitor-metrics/fe-metrics.md +++ b/docs/en/administrator-guide/operation/monitor-metrics/fe-metrics.md @@ -75,3 +75,51 @@ Value of the `Tcp: OutSegs` field in `/proc/net/snmp`. Represents the number of Use `(NEW_tcp_retrans_segs - OLD_tcp_retrans_segs) / (NEW_tcp_out_segs - OLD_tcp_out_segs)` can calculate the retrans rate of TCP packets. Usually used to troubleshoot network problems. + +### `jvm_thread{type="count"}` + +Value of the `count` type in `jvm_thread`. Represents the current number of live threads including both daemon and non-daemon threads. + +Usually used to troubleshoot jvm threads problems for FE. + +### `jvm_thread{type="peak_count"}` + +Value of the `peak_count` type in `jvm_thread`. Represents the current number of live threads including both daemon and non-daemon threads. + +Usually used to troubleshoot jvm threads problems for FE. + +### `jvm_thread{type="new_count"}` + +Value of the `new_count` type in `jvm_thread`. Represents the current number of threads which state is NEW. + +Usually used to troubleshoot jvm threads problems for FE. + +### `jvm_thread{type="runnable_count"}` + +Value of the `runnable_count` type in `jvm_thread`. Represents the current number of threads which state is RUNNABLE. + +Usually used to troubleshoot jvm threads problems for FE. + +### `jvm_thread{type="blocked_count"}` + +Value of the `blocked_count` type in `jvm_thread`. Represents the current number of threads which state is BLOCKED. + +Usually used to troubleshoot jvm threads problems for FE. + +### `jvm_thread{type="waiting_count"}` + +Value of the `waiting_count` type in `jvm_thread`. Represents the current number of threads which state is WAITING. + +Usually used to troubleshoot jvm threads problems for FE. + +### `jvm_thread{type="timed_waiting_count"}` + +Value of the `timed_waiting_count` type in `jvm_thread`. Represents the current number of threads which state is TIMED_WAITING. + +Usually used to troubleshoot jvm threads problems for FE. + +### `jvm_thread{type="terminated_count"}` + +Value of the `terminated_count` type in `jvm_thread`. Represents the current number of threads which state is TERMINATED. + +Usually used to troubleshoot jvm threads problems for FE. \ No newline at end of file diff --git a/docs/zh-CN/administrator-guide/operation/monitor-metrics/fe-metrics.md b/docs/zh-CN/administrator-guide/operation/monitor-metrics/fe-metrics.md index aaa18541047d0b..8b58e9f53e9adb 100644 --- a/docs/zh-CN/administrator-guide/operation/monitor-metrics/fe-metrics.md +++ b/docs/zh-CN/administrator-guide/operation/monitor-metrics/fe-metrics.md @@ -75,3 +75,51 @@ FE 的监控项可以通过以下方式访问： 通过 `(NEW_tcp_tcp_retrans_segs - OLD_tcp_retrans_segs) / (NEW_tcp_out_segs - OLD_tcp_out_segs)` 可以计算 TCP 重传率。 通常用于排查网络问题。 + +### `jvm_thread{type="count"}` + +该监控项表示FE节点当前JVM总的线程数量，包含daemon线程和非daemon线程。 + +通常用于排查FE节点的JVM线程运行问题。 + +### `jvm_thread{type="peak_count"}` + +该监控项表示FE节点从JVM启动以来的最大峰值线程数量。 + +通常用于排查FE节点的JVM线程运行问题。 + +### `jvm_thread{type="new_count"}` + +该监控项表示FE节点JVM中处于NEW状态的线程数量。 + +通常用于排查FE节点的JVM线程运行问题。 + +### `jvm_thread{type="runnable_count"}` + +该监控项表示FE节点JVM中处于RUNNABLE状态的线程数量。 + +通常用于排查FE节点的JVM线程运行问题。 + +### `jvm_thread{type="blocked_count"}` + +该监控项表示FE节点JVM中处于BLOCKED状态的线程数量。 + +通常用于排查FE节点的JVM线程运行问题。 + +### `jvm_thread{type="waiting_count"}` + +该监控项表示FE节点JVM中处于WAITING状态的线程数量。 + +通常用于排查FE节点的JVM线程运行问题。 + +### `jvm_thread{type="timed_waiting_count"}` + +该监控项表示FE节点JVM中处于TIMED_WAITING状态的线程数量。 + +通常用于排查FE节点的JVM线程运行问题。 + +### `jvm_thread{type="terminated_count"}` + +该监控项表示FE节点JVM中处于TERMINATED状态的线程数量。 + +通常用于排查FE节点的JVM线程运行问题。 \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/JvmProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/JvmProcDir.java index ca3cffa6844a62..fce113f745c26c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/JvmProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/JvmProcDir.java @@ -94,6 +94,12 @@ public ProcResult fetchResult() throws AnalysisException { Threads threads = jvmStats.getThreads(); result.addRow(genRow("threads count", threads.getCount())); result.addRow(genRow("threads peak count", threads.getPeakCount())); + result.addRow(genRow("threads new count", threads.getThreadsNewCount())); + result.addRow(genRow("threads runnable count", threads.getThreadsRunnableCount())); + result.addRow(genRow("threads blocked count", threads.getThreadsBlockedCount())); + result.addRow(genRow("threads waiting count", threads.getThreadsWaitingCount())); + result.addRow(genRow("threads timed_waiting count", threads.getThreadsTimedWaitingCount())); + result.addRow(genRow("threads terminated count", threads.getThreadsTerminatedCount())); return result; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java index 6deba5bb71294c..c692ebcf997865 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java @@ -134,6 +134,12 @@ public void visitJvm(StringBuilder sb, JvmStats jvmStats) { sb.append(Joiner.on(" ").join(TYPE, JVM_THREAD, "gauge\n")); sb.append(JVM_THREAD).append("{type=\"count\"} ").append(threads.getCount()).append("\n"); sb.append(JVM_THREAD).append("{type=\"peak_count\"} ").append(threads.getPeakCount()).append("\n"); + sb.append(JVM_THREAD).append("{type=\"new_count\"} ").append(threads.getThreadsNewCount()).append("\n"); + sb.append(JVM_THREAD).append("{type=\"runnable_count\"} ").append(threads.getThreadsRunnableCount()).append("\n"); + sb.append(JVM_THREAD).append("{type=\"blocked_count\"} ").append(threads.getThreadsBlockedCount()).append("\n"); + sb.append(JVM_THREAD).append("{type=\"waiting_count\"} ").append(threads.getThreadsWaitingCount()).append("\n"); + sb.append(JVM_THREAD).append("{type=\"timed_waiting_count\"} ").append(threads.getThreadsTimedWaitingCount()).append("\n"); + sb.append(JVM_THREAD).append("{type=\"terminated_count\"} ").append(threads.getThreadsTerminatedCount()).append("\n"); return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/JvmStats.java b/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/JvmStats.java index a758f36637a362..a298759938a8d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/JvmStats.java +++ b/fe/fe-core/src/main/java/org/apache/doris/monitor/jvm/JvmStats.java @@ -29,6 +29,7 @@ import java.lang.management.MemoryUsage; import java.lang.management.RuntimeMXBean; import java.lang.management.ThreadMXBean; +import java.lang.management.ThreadInfo; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -82,7 +83,28 @@ public static JvmStats jvmStats() { } Mem mem = new Mem(heapCommitted, heapUsed, heapMax, nonHeapCommitted, nonHeapUsed, Collections.unmodifiableList(pools)); - Threads threads = new Threads(threadMXBean.getThreadCount(), threadMXBean.getPeakThreadCount()); + + int threadsNew = 0; + int threadsRunnable = 0; + int threadsBlocked = 0; + int threadsWaiting = 0; + int threadsTimedWaiting = 0; + int threadsTerminated = 0; + long threadIds[] = threadMXBean.getAllThreadIds(); + for (ThreadInfo threadInfo : threadMXBean.getThreadInfo(threadIds, 0)) { + if (threadInfo == null) continue; // race protection + switch (threadInfo.getThreadState()) { + case NEW: threadsNew++; break; + case RUNNABLE: threadsRunnable++; break; + case BLOCKED: threadsBlocked++; break; + case WAITING: threadsWaiting++; break; + case TIMED_WAITING: threadsTimedWaiting++; break; + case TERMINATED: threadsTerminated++; break; + default: break; + } + } + Threads threads = new Threads(threadMXBean.getThreadCount(), threadMXBean.getPeakThreadCount(), threadsNew, + threadsRunnable, threadsBlocked, threadsWaiting, threadsTimedWaiting, threadsTerminated); List gcMxBeans = ManagementFactory.getGarbageCollectorMXBeans(); GarbageCollector[] collectors = new GarbageCollector[gcMxBeans.size()]; @@ -192,6 +214,12 @@ static final class Fields { static final String THREADS = "threads"; static final String COUNT = "count"; static final String PEAK_COUNT = "peak_count"; + static final String NEW_COUNT = "new_count"; + static final String RUNNING_COUNT = "runnable_count"; + static final String BLOCKED_COUNT = "blocked_count"; + static final String WAITING_COUNT = "waiting_count"; + static final String TIMED_WAITING_COUNT = "timed_waiting_count"; + static final String TERMINATED_COUNT = "terminated_count"; static final String GC = "gc"; static final String COLLECTORS = "collectors"; @@ -264,10 +292,23 @@ public static class Threads { private final int count; private final int peakCount; - - public Threads(int count, int peakCount) { + private final int threadsNewCount; + private final int threadsRunnableCount; + private final int threadsBlockedCount; + private final int threadsWaitingCount; + private final int threadsTimedWaitingCount; + private final int threadsTerminatedCount; + + public Threads(int count, int peakCount, int threadsNewCount, int threadsRunnableCount, int threadsBlockedCount, + int threadsWaitingCount, int threadsTimedWaitingCount, int threadsTerminatedCount) { this.count = count; this.peakCount = peakCount; + this.threadsNewCount = threadsNewCount; + this.threadsRunnableCount = threadsRunnableCount; + this.threadsBlockedCount = threadsBlockedCount; + this.threadsWaitingCount = threadsWaitingCount; + this.threadsTimedWaitingCount = threadsTimedWaitingCount; + this.threadsTerminatedCount = threadsTerminatedCount; } public int getCount() { @@ -278,6 +319,30 @@ public int getPeakCount() { return peakCount; } + public int getThreadsNewCount() { + return threadsNewCount; + } + + public int getThreadsRunnableCount() { + return threadsRunnableCount; + } + + public int getThreadsBlockedCount() { + return threadsBlockedCount; + } + + public int getThreadsWaitingCount() { + return threadsWaitingCount; + } + + public int getThreadsTimedWaitingCount() { + return threadsTimedWaitingCount; + } + + public int getThreadsTerminatedCount() { + return threadsTerminatedCount; + } + @Override public String toString() { StringBuilder sb = new StringBuilder();