From 0bcedea245ca00628f6eff8e973ff68a9a7ccb86 Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Wed, 8 Oct 2025 18:25:58 +0800 Subject: [PATCH 01/16] Display CPU and memory information on web console --- docs/querying/sql-metadata-tables.md | 2 + .../java/org/apache/druid/utils/JvmUtils.java | 7 ++++ .../druid/discovery/DiscoveryDruidNode.java | 36 +++++++++++++++-- .../sql/calcite/schema/SystemSchema.java | 14 +++++-- web-console/src/utils/general.tsx | 4 +- .../src/views/services-view/services-view.tsx | 39 ++++++++++++++++++- 6 files changed, 92 insertions(+), 10 deletions(-) diff --git a/docs/querying/sql-metadata-tables.md b/docs/querying/sql-metadata-tables.md index aa1804e29d56..51ce30dff23b 100644 --- a/docs/querying/sql-metadata-tables.md +++ b/docs/querying/sql-metadata-tables.md @@ -237,6 +237,8 @@ Servers table lists all discovered servers in the cluster. |is_leader|BIGINT|1 if the server is currently the 'leader' (for services which have the concept of leadership), otherwise 0 if the server is not the leader, or null if the server type does not have the concept of leadership| |start_time|STRING|Timestamp in ISO8601 format when the server was announced in the cluster| |version|VARCHAR|Druid version running on the server| +|available_processors|BIGINT|Total number of processors available to the server| +|total_memory|BIGINT|Total memory in binary bytes available to the server| To retrieve information about all servers, use the query: ```sql diff --git a/processing/src/main/java/org/apache/druid/utils/JvmUtils.java b/processing/src/main/java/org/apache/druid/utils/JvmUtils.java index c5c502ca156f..95073773542a 100644 --- a/processing/src/main/java/org/apache/druid/utils/JvmUtils.java +++ b/processing/src/main/java/org/apache/druid/utils/JvmUtils.java @@ -21,6 +21,7 @@ import com.google.common.primitives.Ints; import com.google.inject.Inject; +import com.sun.management.OperatingSystemMXBean; import java.io.File; import java.lang.management.ManagementFactory; @@ -44,6 +45,7 @@ public class JvmUtils private static RuntimeInfo RUNTIME_INFO = new RuntimeInfo(); private static final ThreadMXBean THREAD_MX_BEAN = ManagementFactory.getThreadMXBean(); + private static final OperatingSystemMXBean OPERATING_SYSTEM_MX_BEAN = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); private static int computeMajorVersion() { @@ -139,4 +141,9 @@ public static List systemClassPath() ).collect(Collectors.toList()); return jobURLs; } + + public static long getTotalMemory() + { + return OPERATING_SYSTEM_MX_BEAN.getTotalPhysicalMemorySize(); + } } diff --git a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java index 3a62e5344b11..0d1b641f0478 100644 --- a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java +++ b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java @@ -25,6 +25,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Maps; +import com.google.inject.Inject; + import org.apache.druid.client.DruidServer; import org.apache.druid.jackson.StringObjectPairList; import org.apache.druid.java.util.common.DateTimes; @@ -32,6 +34,8 @@ import org.apache.druid.java.util.common.NonnullPair; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.server.DruidNode; +import org.apache.druid.utils.JvmUtils; +import org.apache.druid.utils.RuntimeInfo; import org.joda.time.DateTime; import javax.annotation.Nullable; @@ -54,6 +58,8 @@ public class DiscoveryDruidNode private final DruidNode druidNode; private final NodeRole nodeRole; private final DateTime startTime; + private final Integer availableProcessors; + private final Long totalMemory; /** * Map of service name -> DruidServices. @@ -65,20 +71,24 @@ public class DiscoveryDruidNode */ private final Map services = new HashMap<>(); + @Inject public DiscoveryDruidNode( DruidNode druidNode, NodeRole nodeRole, - Map services + Map services, + RuntimeInfo runtimeInfo ) { - this(druidNode, nodeRole, services, DateTimes.nowUtc()); + this(druidNode, nodeRole, services, DateTimes.nowUtc(), runtimeInfo.getAvailableProcessors(), JvmUtils.getTotalMemory()); } public DiscoveryDruidNode( DruidNode druidNode, NodeRole nodeRole, Map services, - DateTime startTime + DateTime startTime, + Integer availableProcessors, + Long totalMemory ) { this.druidNode = druidNode; @@ -88,6 +98,10 @@ public DiscoveryDruidNode( this.services.putAll(services); } this.startTime = startTime; + + // Happens if service is running older version of Druid + this.availableProcessors = availableProcessors != null ? availableProcessors : -1; + this.totalMemory = totalMemory != null ? totalMemory : -1; } @JsonCreator @@ -96,6 +110,8 @@ private static DiscoveryDruidNode fromJson( @JsonProperty("nodeType") NodeRole nodeRole, @JsonProperty("services") Map rawServices, @JsonProperty("startTime") DateTime startTime, + @JsonProperty("availableProcessors") Integer availableProcessors, + @JsonProperty("totalMemory") Long totalMemory, @JacksonInject ObjectMapper jsonMapper ) { @@ -111,7 +127,7 @@ private static DiscoveryDruidNode fromJson( } } } - return new DiscoveryDruidNode(druidNode, nodeRole, services, startTime); + return new DiscoveryDruidNode(druidNode, nodeRole, services, startTime, availableProcessors, totalMemory); } /** @@ -188,6 +204,18 @@ public DateTime getStartTime() return startTime; } + @JsonProperty + public Integer getAvailableProcessors() + { + return availableProcessors; + } + + @JsonProperty + public Long getTotalMemory() + { + return totalMemory; + } + @Nullable @JsonIgnore public T getService(String key, Class clazz) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 932404f21e78..3e14e3ef4f9f 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -181,6 +181,8 @@ public class SystemSchema extends AbstractSchema .add("is_leader", ColumnType.LONG) .add("start_time", ColumnType.STRING) .add("version", ColumnType.STRING) + .add("available_processors", ColumnType.LONG) + .add("total_memory", ColumnType.LONG) .build(); static final RowSignature SERVER_SEGMENTS_SIGNATURE = RowSignature @@ -641,7 +643,9 @@ private static Object[] buildRowForNonDataServer(DiscoveryDruidNode discoveryDru UNKNOWN_SIZE, null, toStringOrNull(discoveryDruidNode.getStartTime()), - node.getVersion() + node.getVersion(), + (long) discoveryDruidNode.getAvailableProcessors(), + discoveryDruidNode.getTotalMemory() }; } @@ -665,7 +669,9 @@ private static Object[] buildRowForNonDataServerWithLeadership( UNKNOWN_SIZE, isLeader ? 1L : 0L, toStringOrNull(discoveryDruidNode.getStartTime()), - node.getVersion() + node.getVersion(), + (long) discoveryDruidNode.getAvailableProcessors(), + discoveryDruidNode.getTotalMemory() }; } @@ -701,7 +707,9 @@ private static Object[] buildRowForDiscoverableDataServer( druidServerToUse.getMaxSize(), null, toStringOrNull(discoveryDruidNode.getStartTime()), - node.getVersion() + node.getVersion(), + (long) discoveryDruidNode.getAvailableProcessors(), + discoveryDruidNode.getTotalMemory() }; } diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index 3547738ef80d..5686b3a68411 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -292,8 +292,8 @@ export function formatRate(n: NumberLike) { return numeral(n).format('0,0.0') + '/s'; } -export function formatBytes(n: NumberLike): string { - return numeral(n).format('0.00 b'); +export function formatBytes(n: NumberLike, useBinaryBytes: boolean = false): string { + return numeral(n).format(useBinaryBytes ? '0.00 ib' : '0.00 b'); } export function formatByteRate(n: NumberLike): string { diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index 1ac4e42c712a..c96f36e86a0e 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -85,6 +85,8 @@ const TABLE_COLUMNS_BY_MODE: Record '', }, + { + Header: 'CPU processors', + show: visibleColumns.shown('CPU processors'), + accessor: 'available_processors', + className: 'padded', + filterable: false, + width: 120, + Cell: ({ value }) => value, + Aggregated: ({ subRows }) => { + const originalRows: ServiceResultRow[] = subRows.map(r => r._original); + const totalAvailableProcessors = sum(originalRows, s => s.available_processors); + return totalAvailableProcessors; + }, + }, + { + Header: 'Total memory', + show: visibleColumns.shown('Total memory'), + accessor: 'total_memory', + className: 'padded', + width: 120, + filterable: false, + Cell: ({ value }) => { + if (value === null) return ''; + return formatBytes(value, true); + }, + Aggregated: ({ subRows }) => { + const originalRows = subRows.map(r => r._original); + const totalMemory = sum(originalRows, s => s.total_memory); + return formatBytes(totalMemory, true); + }, + }, { Header: 'Detail', show: visibleColumns.shown('Detail'), From 9ab064d89673273171f3a4096b17ab6fb4826fe5 Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:33:19 +0800 Subject: [PATCH 02/16] Add equality checks --- .../druid/discovery/DiscoveryDruidNode.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java index 0d1b641f0478..c53d05abce29 100644 --- a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java +++ b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java @@ -24,9 +24,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import com.google.inject.Inject; - import org.apache.druid.client.DruidServer; import org.apache.druid.jackson.StringObjectPairList; import org.apache.druid.java.util.common.DateTimes; @@ -71,6 +71,19 @@ public class DiscoveryDruidNode */ private final Map services = new HashMap<>(); + /** + * Constructor for tests. In production, the @Inject constructor is used instead. + */ + @VisibleForTesting + public DiscoveryDruidNode( + DruidNode druidNode, + NodeRole nodeRole, + Map services + ) + { + this(druidNode, nodeRole, services, DateTimes.nowUtc(), JvmUtils.getRuntimeInfo().getAvailableProcessors(), JvmUtils.getTotalMemory()); + } + @Inject public DiscoveryDruidNode( DruidNode druidNode, @@ -263,13 +276,15 @@ public boolean equals(Object o) DiscoveryDruidNode that = (DiscoveryDruidNode) o; return Objects.equals(druidNode, that.druidNode) && Objects.equals(nodeRole, that.nodeRole) && - Objects.equals(services, that.services); + Objects.equals(services, that.services) && + Objects.equals(availableProcessors, that.availableProcessors) && + Objects.equals(totalMemory, that.totalMemory); } @Override public int hashCode() { - return Objects.hash(druidNode, nodeRole, services); + return Objects.hash(druidNode, nodeRole, services, availableProcessors, totalMemory); } @Override @@ -280,6 +295,8 @@ public String toString() ", nodeRole='" + nodeRole + '\'' + ", services=" + services + '\'' + ", startTime=" + startTime + + ", availableProcessors=" + availableProcessors + + ", totalMemory=" + totalMemory + '}'; } } From 0b885fea5fc2cb49ae09638c650810f39352d944 Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:34:32 +0800 Subject: [PATCH 03/16] Update to use new DiscoveryDruidNode constructor --- .../seekablestream/SeekableStreamIndexTaskRunner.java | 3 ++- .../src/main/java/org/apache/druid/cli/ServerRunnable.java | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java index c6e4a9ff2356..760a8bf5cab5 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java @@ -445,7 +445,8 @@ private TaskStatus runInternal(TaskToolbox toolbox) throws Exception ImmutableMap.of( toolbox.getDataNodeService().getName(), toolbox.getDataNodeService(), lookupNodeService.getName(), lookupNodeService - ) + ), + toolbox.getAdjustedRuntimeInfo() ); Throwable caughtExceptionOuter = null; diff --git a/services/src/main/java/org/apache/druid/cli/ServerRunnable.java b/services/src/main/java/org/apache/druid/cli/ServerRunnable.java index 3dc1711a50e6..b1ef49f09678 100644 --- a/services/src/main/java/org/apache/druid/cli/ServerRunnable.java +++ b/services/src/main/java/org/apache/druid/cli/ServerRunnable.java @@ -41,6 +41,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.server.DruidNode; +import org.apache.druid.utils.RuntimeInfo; import java.lang.annotation.Annotation; import java.util.Collections; @@ -155,6 +156,7 @@ private DiscoverySideEffectsProvider(final boolean useLegacyAnnouncer) @Override public Child get() { + final RuntimeInfo runtimeInfo = injector.getInstance(RuntimeInfo.class); for (NodeRole nodeRole : nodeRoles) { ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); for (Class clazz : serviceClasses.getOrDefault(nodeRole, Collections.emptySet())) { @@ -168,7 +170,8 @@ public Child get() ); } } - DiscoveryDruidNode discoveryDruidNode = new DiscoveryDruidNode(druidNode, nodeRole, builder.build()); + DiscoveryDruidNode discoveryDruidNode = new DiscoveryDruidNode(druidNode, nodeRole, builder.build(), + runtimeInfo); lifecycle.addHandler( new Lifecycle.Handler() From 942c046fc843f1f18959f63be5923213e23706da Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:34:55 +0800 Subject: [PATCH 04/16] Add documentation for getTotalMemory method --- .../src/main/java/org/apache/druid/utils/JvmUtils.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/processing/src/main/java/org/apache/druid/utils/JvmUtils.java b/processing/src/main/java/org/apache/druid/utils/JvmUtils.java index 95073773542a..57814fe23723 100644 --- a/processing/src/main/java/org/apache/druid/utils/JvmUtils.java +++ b/processing/src/main/java/org/apache/druid/utils/JvmUtils.java @@ -142,6 +142,12 @@ public static List systemClassPath() return jobURLs; } + /** + * Get the total memory of the machine it is running on. This function is container aware. + * If the machine is running in a container, the function will return the total memory of the container. + * If the machine is not running in a container, the function will return the total memory of the machine. + * @return the total memory of the machine it is running on in bytes. + */ public static long getTotalMemory() { return OPERATING_SYSTEM_MX_BEAN.getTotalPhysicalMemorySize(); From 8fd767dccc1c073c65558958711e253fc08ce41a Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:35:10 +0800 Subject: [PATCH 05/16] Update DiscoveryDruidNode tests --- .../discovery/DiscoveryDruidNodeTest.java | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java b/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java index 0e55e5b68eb2..8ac15363f976 100644 --- a/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java +++ b/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java @@ -58,7 +58,7 @@ public DiscoveryDruidNodeTest() public void testEquals() { EqualsVerifier.forClass(DiscoveryDruidNode.class) - .withNonnullFields("druidNode", "nodeRole", "services") + .withNonnullFields("druidNode", "nodeRole", "services", "availableProcessors", "totalMemory") .withIgnoredFields("startTime") .usingGetClass() .verify(); @@ -155,7 +155,9 @@ public void testDeserializeWithDataNodeServiceWithAWrongPropertyOrder() throws J + " \"serverType\" : \"broker\",\n" + " \"priority\" : 0\n" + " }\n" - + " }\n" + + " },\n" + + " \"availableProcessors\" : 3,\n" + + " \"totalMemory\" : 1234\n" + "}"; Assert.assertEquals( new DiscoveryDruidNode( @@ -173,7 +175,10 @@ public void testDeserializeWithDataNodeServiceWithAWrongPropertyOrder() throws J ImmutableMap.of( "dataNodeService", new DataNodeService("_default_tier", 1000000000, ServerType.BROKER, 0) - ) + ), + null, + 3, + 1234L ), mapper.readValue(json, DiscoveryDruidNode.class) ); @@ -204,7 +209,9 @@ public void testDeserialize_duplicateProperties_shouldSucceedToDeserialize() thr + " \"serverType\" : \"broker\",\n" + " \"priority\" : 0\n" + " }\n" - + " }\n" + + " },\n" + + " \"availableProcessors\" : 6,\n" + + " \"totalMemory\" : 5432\n" + "}"; Assert.assertEquals( new DiscoveryDruidNode( @@ -222,7 +229,10 @@ public void testDeserialize_duplicateProperties_shouldSucceedToDeserialize() thr ImmutableMap.of( "dataNodeService", new DataNodeService("_default_tier", 1000000000, ServerType.BROKER, 0) - ) + ), + null, + 6, + 5432L ), mapper.readValue(json, DiscoveryDruidNode.class) ); @@ -254,7 +264,9 @@ public void testDeserialize_duplicateKeysWithDifferentValus_shouldIgnoreDataNode + " \"serverType\" : \"broker\",\n" + " \"priority\" : 0\n" + " }\n" - + " }\n" + + " },\n" + + " \"availableProcessors\" : 4,\n" + + " \"totalMemory\" : 246810\n" + "}"; Assert.assertEquals( new DiscoveryDruidNode( @@ -269,7 +281,10 @@ public void testDeserialize_duplicateKeysWithDifferentValus_shouldIgnoreDataNode true ), NodeRole.BROKER, - ImmutableMap.of() + ImmutableMap.of(), + null, + 4, + 246810L ), mapper.readValue(json, DiscoveryDruidNode.class) ); From ccfd83d6ca1950384aa3c9937f287f416608309c Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:35:21 +0800 Subject: [PATCH 06/16] Update services view snapshot --- .../__snapshots__/services-view.spec.tsx.snap | 22 +++++++++++++++++++ .../src/views/services-view/services-view.tsx | 2 ++ 2 files changed, 24 insertions(+) diff --git a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap index b21581166968..0c2cb91f71e5 100644 --- a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap +++ b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap @@ -59,6 +59,8 @@ exports[`ServicesView renders data 1`] = ` "Usage", "Start time", "Version", + "CPU processors", + "Total memory", "Detail", ] } @@ -217,6 +219,26 @@ exports[`ServicesView renders data 1`] = ` "show": true, "width": 200, }, + { + "Aggregated": [Function], + "Cell": [Function], + "Header": "CPU processors", + "accessor": "available_processors", + "className": "padded", + "filterable": false, + "show": true, + "width": 120, + }, + { + "Aggregated": [Function], + "Cell": [Function], + "Header": "Total memory", + "accessor": "total_memory", + "className": "padded", + "filterable": false, + "show": true, + "width": 120, + }, { "Aggregated": [Function], "Cell": [Function], diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index c96f36e86a0e..bf7025e19cc2 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -298,6 +298,8 @@ ORDER BY start_time: '1970:01:01T00:00:00Z', is_leader: 0, version: '', + available_processors: -1, + total_memory: -1, }; }, ); From 015e69dd1f00d5e8710328509feb9cd3e39c9aa7 Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Thu, 9 Oct 2025 10:37:11 +0800 Subject: [PATCH 07/16] Run autofix --- web-console/src/utils/general.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index 5686b3a68411..f206ed024575 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -292,7 +292,7 @@ export function formatRate(n: NumberLike) { return numeral(n).format('0,0.0') + '/s'; } -export function formatBytes(n: NumberLike, useBinaryBytes: boolean = false): string { +export function formatBytes(n: NumberLike, useBinaryBytes = false): string { return numeral(n).format(useBinaryBytes ? '0.00 ib' : '0.00 b'); } From 61a9eb6cfc073b2bb81dcefca36c4f041e248cb4 Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Thu, 9 Oct 2025 16:51:13 +0800 Subject: [PATCH 08/16] Use JvmUtils instead --- .../SeekableStreamIndexTaskRunner.java | 3 +-- .../druid/discovery/DiscoveryDruidNode.java | 15 ++++++--------- .../java/org/apache/druid/cli/ServerRunnable.java | 5 +---- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java index 760a8bf5cab5..c6e4a9ff2356 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskRunner.java @@ -445,8 +445,7 @@ private TaskStatus runInternal(TaskToolbox toolbox) throws Exception ImmutableMap.of( toolbox.getDataNodeService().getName(), toolbox.getDataNodeService(), lookupNodeService.getName(), lookupNodeService - ), - toolbox.getAdjustedRuntimeInfo() + ) ); Throwable caughtExceptionOuter = null; diff --git a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java index c53d05abce29..7f4e00a7437c 100644 --- a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java +++ b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java @@ -26,7 +26,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; -import com.google.inject.Inject; import org.apache.druid.client.DruidServer; import org.apache.druid.jackson.StringObjectPairList; import org.apache.druid.java.util.common.DateTimes; @@ -35,7 +34,6 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.server.DruidNode; import org.apache.druid.utils.JvmUtils; -import org.apache.druid.utils.RuntimeInfo; import org.joda.time.DateTime; import javax.annotation.Nullable; @@ -71,28 +69,27 @@ public class DiscoveryDruidNode */ private final Map services = new HashMap<>(); - /** + /** * Constructor for tests. In production, the @Inject constructor is used instead. */ @VisibleForTesting public DiscoveryDruidNode( DruidNode druidNode, NodeRole nodeRole, - Map services + Map services, + DateTime startTime ) { - this(druidNode, nodeRole, services, DateTimes.nowUtc(), JvmUtils.getRuntimeInfo().getAvailableProcessors(), JvmUtils.getTotalMemory()); + this(druidNode, nodeRole, services, startTime, JvmUtils.getRuntimeInfo().getAvailableProcessors(), JvmUtils.getTotalMemory()); } - @Inject public DiscoveryDruidNode( DruidNode druidNode, NodeRole nodeRole, - Map services, - RuntimeInfo runtimeInfo + Map services ) { - this(druidNode, nodeRole, services, DateTimes.nowUtc(), runtimeInfo.getAvailableProcessors(), JvmUtils.getTotalMemory()); + this(druidNode, nodeRole, services, DateTimes.nowUtc(), JvmUtils.getRuntimeInfo().getAvailableProcessors(), JvmUtils.getTotalMemory()); } public DiscoveryDruidNode( diff --git a/services/src/main/java/org/apache/druid/cli/ServerRunnable.java b/services/src/main/java/org/apache/druid/cli/ServerRunnable.java index b1ef49f09678..3dc1711a50e6 100644 --- a/services/src/main/java/org/apache/druid/cli/ServerRunnable.java +++ b/services/src/main/java/org/apache/druid/cli/ServerRunnable.java @@ -41,7 +41,6 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.server.DruidNode; -import org.apache.druid.utils.RuntimeInfo; import java.lang.annotation.Annotation; import java.util.Collections; @@ -156,7 +155,6 @@ private DiscoverySideEffectsProvider(final boolean useLegacyAnnouncer) @Override public Child get() { - final RuntimeInfo runtimeInfo = injector.getInstance(RuntimeInfo.class); for (NodeRole nodeRole : nodeRoles) { ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); for (Class clazz : serviceClasses.getOrDefault(nodeRole, Collections.emptySet())) { @@ -170,8 +168,7 @@ public Child get() ); } } - DiscoveryDruidNode discoveryDruidNode = new DiscoveryDruidNode(druidNode, nodeRole, builder.build(), - runtimeInfo); + DiscoveryDruidNode discoveryDruidNode = new DiscoveryDruidNode(druidNode, nodeRole, builder.build()); lifecycle.addHandler( new Lifecycle.Handler() From 6cd4542cbbe2190712e7adbdb25e20cd4eede64f Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Thu, 9 Oct 2025 16:51:19 +0800 Subject: [PATCH 09/16] Update doc wording --- docs/querying/sql-metadata-tables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/querying/sql-metadata-tables.md b/docs/querying/sql-metadata-tables.md index 51ce30dff23b..33b342204650 100644 --- a/docs/querying/sql-metadata-tables.md +++ b/docs/querying/sql-metadata-tables.md @@ -238,7 +238,7 @@ Servers table lists all discovered servers in the cluster. |start_time|STRING|Timestamp in ISO8601 format when the server was announced in the cluster| |version|VARCHAR|Druid version running on the server| |available_processors|BIGINT|Total number of processors available to the server| -|total_memory|BIGINT|Total memory in binary bytes available to the server| +|total_memory|BIGINT|Total memory in bytes available to the server| To retrieve information about all servers, use the query: ```sql From 37ae2b17187a24fbf604cc0b7954bac03be21f97 Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Thu, 9 Oct 2025 17:11:05 +0800 Subject: [PATCH 10/16] Update .spelling --- docs/querying/sql-metadata-tables.md | 1 + website/.spelling | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/querying/sql-metadata-tables.md b/docs/querying/sql-metadata-tables.md index 33b342204650..cf128f4eaeb8 100644 --- a/docs/querying/sql-metadata-tables.md +++ b/docs/querying/sql-metadata-tables.md @@ -239,6 +239,7 @@ Servers table lists all discovered servers in the cluster. |version|VARCHAR|Druid version running on the server| |available_processors|BIGINT|Total number of processors available to the server| |total_memory|BIGINT|Total memory in bytes available to the server| + To retrieve information about all servers, use the query: ```sql diff --git a/website/.spelling b/website/.spelling index 767ef4249aba..af214ed11c8f 100644 --- a/website/.spelling +++ b/website/.spelling @@ -711,6 +711,8 @@ is_realtime java.sql.Types last_compaction_state max_size +available_processors +total_memory num_replicas num_rows num_segments From c67929bcb5424eee6918cd5d2032fbfce29cecf1 Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Thu, 9 Oct 2025 17:27:08 +0800 Subject: [PATCH 11/16] Update server field size in test --- .../sql/calcite/schema/SystemSchemaTest.java | 73 ++++++++++++++----- 1 file changed, 55 insertions(+), 18 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index d523d970fccf..2d0fa3185c01 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -111,6 +111,7 @@ import org.apache.druid.timeline.SegmentStatusInCluster; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.apache.druid.timeline.partition.ShardSpec; +import org.apache.druid.utils.JvmUtils; import org.easymock.EasyMock; import org.jboss.netty.handler.codec.http.HttpResponse; import org.joda.time.DateTime; @@ -381,6 +382,8 @@ public void setUp(@TempDir File tmpDir) throws Exception final List realtimeSegments = ImmutableList.of(segment2, segment4, segment5); private final DateTime startTime = DateTimes.nowUtc(); + private final long availableProcessors = JvmUtils.getRuntimeInfo().getAvailableProcessors(); + private final long totalMemory = JvmUtils.getTotalMemory(); private final String version = GuavaUtils.firstNonNull( SystemSchemaTest.class.getPackage().getImplementationVersion(), @@ -558,7 +561,7 @@ public void testGetTableMap() final SystemSchema.ServersTable serversTable = (SystemSchema.ServersTable) schema.getTableMap().get("servers"); final RelDataType serverRowType = serversTable.getRowType(new JavaTypeFactoryImpl()); final List serverFields = serverRowType.getFieldList(); - Assert.assertEquals(11, serverFields.size()); + Assert.assertEquals(13, serverFields.size()); Assert.assertEquals("server", serverFields.get(0).getName()); Assert.assertEquals(SqlTypeName.VARCHAR, serverFields.get(0).getType().getSqlTypeName()); } @@ -858,7 +861,9 @@ public void testServersTable() throws URISyntaxException 0L, nonLeader, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -873,7 +878,9 @@ public void testServersTable() throws URISyntaxException 1000L, nonLeader, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -888,7 +895,9 @@ public void testServersTable() throws URISyntaxException 1000L, nonLeader, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -903,7 +912,9 @@ public void testServersTable() throws URISyntaxException 1000L, nonLeader, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -918,7 +929,9 @@ public void testServersTable() throws URISyntaxException 1000L, nonLeader, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add(createExpectedRow( @@ -932,7 +945,9 @@ public void testServersTable() throws URISyntaxException 1000L, nonLeader, startTimeStr, - version + version, + availableProcessors, + totalMemory )); expectedRows.add( createExpectedRow( @@ -946,7 +961,9 @@ public void testServersTable() throws URISyntaxException 0L, 1L, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -961,7 +978,9 @@ public void testServersTable() throws URISyntaxException 0L, nonLeader, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -976,7 +995,9 @@ public void testServersTable() throws URISyntaxException 1000L, nonLeader, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -991,7 +1012,9 @@ public void testServersTable() throws URISyntaxException 0L, 1L, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -1006,7 +1029,9 @@ public void testServersTable() throws URISyntaxException 0L, 0L, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -1021,7 +1046,9 @@ public void testServersTable() throws URISyntaxException 0L, 0L, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -1036,7 +1063,9 @@ public void testServersTable() throws URISyntaxException 0L, nonLeader, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -1051,7 +1080,9 @@ public void testServersTable() throws URISyntaxException 0L, nonLeader, startTimeStr, - version + version, + availableProcessors, + totalMemory ) ); expectedRows.add(createExpectedRow( @@ -1065,7 +1096,9 @@ public void testServersTable() throws URISyntaxException 1000L, nonLeader, startTimeStr, - version + version, + availableProcessors, + totalMemory )); Assert.assertEquals(expectedRows.size(), rows.size()); for (int i = 0; i < rows.size(); i++) { @@ -1099,7 +1132,9 @@ private Object[] createExpectedRow( @Nullable Long maxSize, @Nullable Long isLeader, String startTime, - String version + String version, + long availableProcessors, + long totalMemory ) { return new Object[]{ @@ -1113,7 +1148,9 @@ private Object[] createExpectedRow( maxSize, isLeader, startTime, - version + version, + availableProcessors, + totalMemory }; } From c980fc3aa0c53e27c94bd8ef6bcfd1ad6425d9f2 Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Tue, 14 Oct 2025 09:50:55 +0800 Subject: [PATCH 12/16] Fix based on comments --- docs/querying/sql-metadata-tables.md | 2 +- .../java/org/apache/druid/discovery/DiscoveryDruidNode.java | 6 ++++-- web-console/src/views/services-view/services-view.tsx | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/querying/sql-metadata-tables.md b/docs/querying/sql-metadata-tables.md index cf128f4eaeb8..b7c844ef660c 100644 --- a/docs/querying/sql-metadata-tables.md +++ b/docs/querying/sql-metadata-tables.md @@ -237,7 +237,7 @@ Servers table lists all discovered servers in the cluster. |is_leader|BIGINT|1 if the server is currently the 'leader' (for services which have the concept of leadership), otherwise 0 if the server is not the leader, or null if the server type does not have the concept of leadership| |start_time|STRING|Timestamp in ISO8601 format when the server was announced in the cluster| |version|VARCHAR|Druid version running on the server| -|available_processors|BIGINT|Total number of processors available to the server| +|available_processors|BIGINT|Total number of CPU processors available to the server| |total_memory|BIGINT|Total memory in bytes available to the server| To retrieve information about all servers, use the query: diff --git a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java index 7f4e00a7437c..684a3c1776ed 100644 --- a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java +++ b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java @@ -53,6 +53,8 @@ public class DiscoveryDruidNode { private static final Logger LOG = new Logger(DiscoveryDruidNode.class); + private final Integer UNKNOWN_VALUE = -1; + private final DruidNode druidNode; private final NodeRole nodeRole; private final DateTime startTime; @@ -110,8 +112,8 @@ public DiscoveryDruidNode( this.startTime = startTime; // Happens if service is running older version of Druid - this.availableProcessors = availableProcessors != null ? availableProcessors : -1; - this.totalMemory = totalMemory != null ? totalMemory : -1; + this.availableProcessors = availableProcessors != null ? availableProcessors : UNKNOWN_VALUE; + this.totalMemory = totalMemory != null ? totalMemory : UNKNOWN_VALUE; } @JsonCreator diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index bf7025e19cc2..d8e231057307 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -644,7 +644,7 @@ ORDER BY className: 'padded', filterable: false, width: 120, - Cell: ({ value }) => value, + Cell: ({ value }) => (value === null ? '' : value), Aggregated: ({ subRows }) => { const originalRows: ServiceResultRow[] = subRows.map(r => r._original); const totalAvailableProcessors = sum(originalRows, s => s.available_processors); @@ -663,7 +663,7 @@ ORDER BY return formatBytes(value, true); }, Aggregated: ({ subRows }) => { - const originalRows = subRows.map(r => r._original); + const originalRows: ServiceResultRow[] = subRows.map(r => r._original); const totalMemory = sum(originalRows, s => s.total_memory); return formatBytes(totalMemory, true); }, From d1efbcf1086a49386aacbba6d3c057791e8e6e0b Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Tue, 14 Oct 2025 10:59:30 +0800 Subject: [PATCH 13/16] Update unknown value to be static --- .../java/org/apache/druid/discovery/DiscoveryDruidNode.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java index 684a3c1776ed..e4e37bcaa312 100644 --- a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java +++ b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java @@ -52,8 +52,7 @@ public class DiscoveryDruidNode { private static final Logger LOG = new Logger(DiscoveryDruidNode.class); - - private final Integer UNKNOWN_VALUE = -1; + private static final Integer UNKNOWN_VALUE = -1; private final DruidNode druidNode; private final NodeRole nodeRole; From e869da0e53d9b46d080e1414241c697062d14370 Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Tue, 14 Oct 2025 14:33:36 +0800 Subject: [PATCH 14/16] Update integration tests --- .../tests/security/AbstractAuthConfigurationTest.java | 7 ++++--- .../resources/results/auth_test_sys_schema_servers.json | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/integration-tests/src/test/java/org/apache/druid/tests/security/AbstractAuthConfigurationTest.java b/integration-tests/src/test/java/org/apache/druid/tests/security/AbstractAuthConfigurationTest.java index 53803b098136..ee8dda261d8f 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/security/AbstractAuthConfigurationTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/security/AbstractAuthConfigurationTest.java @@ -1026,9 +1026,8 @@ protected void setExpectedSystemSchemaObjects() throws IOException /** * curr_size on historicals changes because cluster state is not isolated across - * different - * integration tests, zero it out for consistent test results - * version and start_time are not configurable therefore we zero them as well + * different integration tests, zero it out for consistent test results. + * version, start_time, available_processors, total_memory are not configurable therefore we zero them as well */ protected static List> getServersWithoutNonConfigurableFields(List> servers) { @@ -1039,6 +1038,8 @@ protected static List> getServersWithoutNonConfigurableField newServer.put("curr_size", 0); newServer.put("start_time", "0"); newServer.put("version", "0.0.0"); + newServer.put("available_processors", 0); + newServer.put("total_memory", 0); return newServer; } ); diff --git a/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json b/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json index ddbd92e3111e..1e7f9474cd3a 100644 --- a/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json +++ b/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json @@ -11,7 +11,9 @@ "is_leader": null, "start_time": "0", "version": "0.0.0", - "labels": null + "labels": null, + "available_processors": 0, + "total_memory": 0 }, { "server": "%%BROKER%%:8282", @@ -25,6 +27,8 @@ "is_leader": null, "start_time": "0", "version": "0.0.0", - "labels": null + "labels": null, + "available_processors": 0, + "total_memory": 0 } ] From fb4207d56c831e0caabc44061c989ccef9b8360f Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Wed, 15 Oct 2025 17:09:53 +0800 Subject: [PATCH 15/16] Use Runtime to get available processors --- .../java/org/apache/druid/discovery/DiscoveryDruidNode.java | 4 ++-- .../org/apache/druid/sql/calcite/schema/SystemSchemaTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java index e4e37bcaa312..875426682c85 100644 --- a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java +++ b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java @@ -81,7 +81,7 @@ public DiscoveryDruidNode( DateTime startTime ) { - this(druidNode, nodeRole, services, startTime, JvmUtils.getRuntimeInfo().getAvailableProcessors(), JvmUtils.getTotalMemory()); + this(druidNode, nodeRole, services, startTime, Runtime.getRuntime().availableProcessors(), JvmUtils.getTotalMemory()); } public DiscoveryDruidNode( @@ -90,7 +90,7 @@ public DiscoveryDruidNode( Map services ) { - this(druidNode, nodeRole, services, DateTimes.nowUtc(), JvmUtils.getRuntimeInfo().getAvailableProcessors(), JvmUtils.getTotalMemory()); + this(druidNode, nodeRole, services, DateTimes.nowUtc(), Runtime.getRuntime().availableProcessors(), JvmUtils.getTotalMemory()); } public DiscoveryDruidNode( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index c3a46ac391ad..3fe3068a370d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -382,7 +382,7 @@ public void setUp(@TempDir File tmpDir) throws Exception final List realtimeSegments = ImmutableList.of(segment2, segment4, segment5); private final DateTime startTime = DateTimes.nowUtc(); - private final long availableProcessors = JvmUtils.getRuntimeInfo().getAvailableProcessors(); + private final long availableProcessors = Runtime.getRuntime().availableProcessors(); private final long totalMemory = JvmUtils.getTotalMemory(); private final String version = GuavaUtils.firstNonNull( From 363c5f189ad79e09d3706ea8bd400f4d7e924549 Mon Sep 17 00:00:00 2001 From: Gabriel Chang <77312579+GabrielCWT@users.noreply.github.com> Date: Tue, 21 Oct 2025 09:37:18 +0800 Subject: [PATCH 16/16] Update unknown value to int --- .../java/org/apache/druid/discovery/DiscoveryDruidNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java index 875426682c85..7d24c203b2e0 100644 --- a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java +++ b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java @@ -52,7 +52,7 @@ public class DiscoveryDruidNode { private static final Logger LOG = new Logger(DiscoveryDruidNode.class); - private static final Integer UNKNOWN_VALUE = -1; + private static final int UNKNOWN_VALUE = -1; private final DruidNode druidNode; private final NodeRole nodeRole;