From 232631f86a5d5a2944970d66c2f783e89862eb49 Mon Sep 17 00:00:00 2001 From: Atul Mohan Date: Fri, 11 Nov 2022 13:49:39 -0800 Subject: [PATCH 1/9] Server start time --- docs/querying/sql-metadata-tables.md | 2 +- .../druid/discovery/DiscoveryDruidNode.java | 30 ++++- .../discovery/DiscoveryDruidNodeTest.java | 1 + .../sql/calcite/schema/SystemSchema.java | 10 +- .../sql/calcite/schema/SystemSchemaTest.java | 103 ++++++++++++------ .../__snapshots__/services-view.spec.tsx.snap | 11 ++ .../services-view/services-view.spec.tsx | 2 + .../src/views/services-view/services-view.tsx | 26 ++++- 8 files changed, 142 insertions(+), 43 deletions(-) diff --git a/docs/querying/sql-metadata-tables.md b/docs/querying/sql-metadata-tables.md index b48b795e3828..d1f13784e9f4 100644 --- a/docs/querying/sql-metadata-tables.md +++ b/docs/querying/sql-metadata-tables.md @@ -207,7 +207,7 @@ Servers table lists all discovered servers in the cluster. |current_size|LONG|Current size of segments in bytes on this server. Only valid for HISTORICAL type, for other types it's 0| |max_size|LONG|Max size in bytes this server recommends to assign to segments see [druid.server.maxSize](../configuration/index.md#historical-general-configuration). Only valid for HISTORICAL type, for other types it's 0| |is_leader|LONG|1 if the server is currently the 'leader' (for services which have the concept of leadership), otherwise 0 if the server is not the leader, or the default long value (0 or null depending on `druid.generic.useDefaultValueForNull`) if the server type does not have the concept of leadership| - +|start_time|STRING|Timestamp in ISO8601 format corresponding to when the server was announced in the cluster| To retrieve information about all servers, use the query: ```sql diff --git a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java index 17be440c8333..5e0386156181 100644 --- a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java +++ b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java @@ -25,10 +25,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Maps; import org.apache.druid.jackson.StringObjectPairList; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.NonnullPair; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.server.DruidNode; +import org.joda.time.DateTime; import java.util.HashMap; import java.util.List; @@ -48,6 +50,7 @@ public class DiscoveryDruidNode private final DruidNode druidNode; private final NodeRole nodeRole; + private final DateTime startTime; /** * Map of service name -> DruidServices. @@ -64,6 +67,16 @@ public DiscoveryDruidNode( NodeRole nodeRole, Map services ) + { + this(druidNode, nodeRole, services, DateTimes.nowUtc()); + } + + public DiscoveryDruidNode( + DruidNode druidNode, + NodeRole nodeRole, + Map services, + DateTime startTime + ) { this.druidNode = druidNode; this.nodeRole = nodeRole; @@ -71,6 +84,7 @@ public DiscoveryDruidNode( if (services != null && !services.isEmpty()) { this.services.putAll(services); } + this.startTime = startTime; } @JsonCreator @@ -78,6 +92,7 @@ private static DiscoveryDruidNode fromJson( @JsonProperty("druidNode") DruidNode druidNode, @JsonProperty("nodeType") NodeRole nodeRole, @JsonProperty("services") Map rawServices, + @JsonProperty("startTime") DateTime startTime, @JacksonInject ObjectMapper jsonMapper ) { @@ -93,7 +108,7 @@ private static DiscoveryDruidNode fromJson( } } } - return new DiscoveryDruidNode(druidNode, nodeRole, services); + return new DiscoveryDruidNode(druidNode, nodeRole, services, startTime); } /** @@ -104,10 +119,10 @@ private static DiscoveryDruidNode fromJson( * This is definitely a bug of DataNodeService, but, since renaming one of those duplicate keys will * break compatibility, DataNodeService still has the deprecated "type" property. * See the Javadoc of DataNodeService for more details. - * + *

* This function catches such duplicate keys and rewrites the deprecated "type" to "serverType", * so that we don't lose any properties. - * + *

* This method can be removed together when we entirely remove the deprecated "type" property from DataNodeService. */ @Deprecated @@ -164,6 +179,12 @@ public DruidNode getDruidNode() return druidNode; } + @JsonProperty + public DateTime getStartTime() + { + return startTime; + } + @Override public boolean equals(Object o) { @@ -191,7 +212,8 @@ public String toString() return "DiscoveryDruidNode{" + "druidNode=" + druidNode + ", nodeRole='" + nodeRole + '\'' + - ", services=" + services + + ", services=" + services + '\'' + + ", startTime=" + startTime + '}'; } } diff --git a/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java b/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java index 64976bd82235..0e55e5b68eb2 100644 --- a/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java +++ b/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java @@ -59,6 +59,7 @@ public void testEquals() { EqualsVerifier.forClass(DiscoveryDruidNode.class) .withNonnullFields("druidNode", "nodeRole", "services") + .withIgnoredFields("startTime") .usingGetClass() .verify(); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 4af0b934ef18..aaf07526c22d 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -164,6 +164,7 @@ public class SystemSchema extends AbstractSchema .add("curr_size", ColumnType.LONG) .add("max_size", ColumnType.LONG) .add("is_leader", ColumnType.LONG) + .add("start_time", ColumnType.STRING) .build(); static final RowSignature SERVER_SEGMENTS_SIGNATURE = RowSignature @@ -594,7 +595,8 @@ private static Object[] buildRowForNonDataServer(DiscoveryDruidNode discoveryDru null, UNKNOWN_SIZE, UNKNOWN_SIZE, - NullHandling.defaultLongValue() + NullHandling.defaultLongValue(), + toStringOrNull(discoveryDruidNode.getStartTime()) }; } @@ -613,7 +615,8 @@ private static Object[] buildRowForNonDataServerWithLeadership(DiscoveryDruidNod null, UNKNOWN_SIZE, UNKNOWN_SIZE, - isLeader ? 1L : 0L + isLeader ? 1L : 0L, + toStringOrNull(discoveryDruidNode.getStartTime()) }; } @@ -647,7 +650,8 @@ private static Object[] buildRowForDiscoverableDataServer( druidServerToUse.getTier(), currentSize, druidServerToUse.getMaxSize(), - NullHandling.defaultLongValue() + NullHandling.defaultLongValue(), + toStringOrNull(discoveryDruidNode.getStartTime()) }; } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index 1ec52a8e94cd..3bc3693d5696 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -49,6 +49,7 @@ import org.apache.druid.discovery.DruidNodeDiscoveryProvider; import org.apache.druid.discovery.NodeRole; import org.apache.druid.indexer.partitions.DynamicPartitionsSpec; +import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Intervals; @@ -105,6 +106,7 @@ import org.jboss.netty.handler.codec.http.HttpResponse; import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.jboss.netty.handler.codec.http.HttpVersion; +import org.joda.time.DateTime; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; @@ -378,40 +380,48 @@ public void setUp() throws Exception final List realtimeSegments = ImmutableList.of(segment2, segment4, segment5); + private final DateTime startTime = DateTimes.nowUtc(); + private final DiscoveryDruidNode coordinator = new DiscoveryDruidNode( new DruidNode("s1", "localhost", false, 8081, null, true, false), NodeRole.COORDINATOR, - ImmutableMap.of() + ImmutableMap.of(), + startTime ); private final DiscoveryDruidNode coordinator2 = new DiscoveryDruidNode( new DruidNode("s1", "localhost", false, 8181, null, true, false), NodeRole.COORDINATOR, - ImmutableMap.of() + ImmutableMap.of(), + startTime ); private final DiscoveryDruidNode overlord = new DiscoveryDruidNode( new DruidNode("s2", "localhost", false, 8090, null, true, false), NodeRole.OVERLORD, - ImmutableMap.of() + ImmutableMap.of(), + startTime ); private final DiscoveryDruidNode overlord2 = new DiscoveryDruidNode( new DruidNode("s2", "localhost", false, 8190, null, true, false), NodeRole.OVERLORD, - ImmutableMap.of() + ImmutableMap.of(), + startTime ); private final DiscoveryDruidNode broker1 = new DiscoveryDruidNode( new DruidNode("s3", "localhost", false, 8082, null, true, false), NodeRole.BROKER, - ImmutableMap.of() + ImmutableMap.of(), + startTime ); private final DiscoveryDruidNode broker2 = new DiscoveryDruidNode( new DruidNode("s3", "brokerHost", false, 8082, null, true, false), NodeRole.BROKER, - ImmutableMap.of() + ImmutableMap.of(), + startTime ); private final DiscoveryDruidNode brokerWithBroadcastSegments = new DiscoveryDruidNode( @@ -419,13 +429,15 @@ public void setUp() throws Exception NodeRole.BROKER, ImmutableMap.of( DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerType.BROKER, 0) - ) + ), + startTime ); private final DiscoveryDruidNode router = new DiscoveryDruidNode( new DruidNode("s4", "localhost", false, 8888, null, true, false), NodeRole.ROUTER, - ImmutableMap.of() + ImmutableMap.of(), + startTime ); private final DiscoveryDruidNode historical1 = new DiscoveryDruidNode( @@ -433,7 +445,8 @@ DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerT NodeRole.HISTORICAL, ImmutableMap.of( DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerType.HISTORICAL, 0) - ) + ), + startTime ); private final DiscoveryDruidNode historical2 = new DiscoveryDruidNode( @@ -441,7 +454,8 @@ DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerT NodeRole.HISTORICAL, ImmutableMap.of( DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerType.HISTORICAL, 0) - ) + ), + startTime ); private final DiscoveryDruidNode lameHistorical = new DiscoveryDruidNode( @@ -449,13 +463,15 @@ DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerT NodeRole.HISTORICAL, ImmutableMap.of( DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerType.HISTORICAL, 0) - ) + ), + startTime ); private final DiscoveryDruidNode middleManager = new DiscoveryDruidNode( new DruidNode("s6", "mmHost", false, 8091, null, true, false), NodeRole.MIDDLE_MANAGER, - ImmutableMap.of() + ImmutableMap.of(), + startTime ); private final DiscoveryDruidNode peon1 = new DiscoveryDruidNode( @@ -463,7 +479,8 @@ DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerT NodeRole.PEON, ImmutableMap.of( DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerType.INDEXER_EXECUTOR, 0) - ) + ), + startTime ); private final DiscoveryDruidNode peon2 = new DiscoveryDruidNode( @@ -471,7 +488,8 @@ DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerT NodeRole.PEON, ImmutableMap.of( DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerType.INDEXER_EXECUTOR, 0) - ) + ), + startTime ); private final DiscoveryDruidNode indexer = new DiscoveryDruidNode( @@ -479,7 +497,8 @@ DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerT NodeRole.INDEXER, ImmutableMap.of( DataNodeService.DISCOVERY_SERVICE_KEY, new DataNodeService("tier", 1000, ServerType.INDEXER_EXECUTOR, 0) - ) + ), + startTime ); private final ImmutableDruidServer druidServer1 = new ImmutableDruidServer( @@ -534,7 +553,7 @@ public void testGetTableMap() final SystemSchema.ServersTable serversTable = (SystemSchema.ServersTable) schema.getTableMap().get("servers"); final RelDataType serverRowType = serversTable.getRowType(new JavaTypeFactoryImpl()); final List serverFields = serverRowType.getFieldList(); - Assert.assertEquals(9, serverFields.size()); + Assert.assertEquals(10, serverFields.size()); Assert.assertEquals("server", serverFields.get(0).getName()); Assert.assertEquals(SqlTypeName.VARCHAR, serverFields.get(0).getType().getSqlTypeName()); } @@ -812,6 +831,7 @@ public void testServersTable() final List expectedRows = new ArrayList<>(); final Long nonLeader = NullHandling.defaultLongValue(); + final String startTimeStr = startTime.toString(); expectedRows.add( createExpectedRow( "brokerHost:8082", @@ -822,7 +842,8 @@ public void testServersTable() null, 0L, 0L, - nonLeader + nonLeader, + startTimeStr ) ); expectedRows.add( @@ -835,7 +856,8 @@ public void testServersTable() "tier", 0L, 1000L, - nonLeader + nonLeader, + startTimeStr ) ); expectedRows.add( @@ -848,7 +870,8 @@ public void testServersTable() "tier", 400L, 1000L, - nonLeader + nonLeader, + startTimeStr ) ); expectedRows.add( @@ -861,7 +884,8 @@ public void testServersTable() "tier", 0L, 1000L, - nonLeader + nonLeader, + startTimeStr ) ); expectedRows.add( @@ -874,7 +898,8 @@ public void testServersTable() "tier", 0L, 1000L, - nonLeader + nonLeader, + startTimeStr ) ); expectedRows.add(createExpectedRow( @@ -886,7 +911,8 @@ public void testServersTable() "tier", 0L, 1000L, - nonLeader + nonLeader, + startTimeStr )); expectedRows.add( createExpectedRow( @@ -898,7 +924,8 @@ public void testServersTable() null, 0L, 0L, - 1L + 1L, + startTimeStr ) ); expectedRows.add( @@ -911,7 +938,8 @@ public void testServersTable() null, 0L, 0L, - nonLeader + nonLeader, + startTimeStr ) ); expectedRows.add( @@ -924,7 +952,8 @@ public void testServersTable() "tier", 200L, 1000L, - nonLeader + nonLeader, + startTimeStr ) ); expectedRows.add( @@ -937,7 +966,8 @@ public void testServersTable() null, 0L, 0L, - 1L + 1L, + startTimeStr ) ); expectedRows.add( @@ -950,7 +980,8 @@ public void testServersTable() null, 0L, 0L, - 0L + 0L, + startTimeStr ) ); expectedRows.add( @@ -963,7 +994,8 @@ public void testServersTable() null, 0L, 0L, - 0L + 0L, + startTimeStr ) ); expectedRows.add( @@ -976,7 +1008,8 @@ public void testServersTable() null, 0L, 0L, - nonLeader + nonLeader, + startTimeStr ) ); expectedRows.add( @@ -989,7 +1022,8 @@ public void testServersTable() null, 0L, 0L, - nonLeader + nonLeader, + startTimeStr ) ); expectedRows.add(createExpectedRow( @@ -1001,7 +1035,8 @@ public void testServersTable() "tier", 0L, 1000L, - nonLeader + nonLeader, + startTimeStr )); Assert.assertEquals(expectedRows.size(), rows.size()); for (int i = 0; i < rows.size(); i++) { @@ -1033,7 +1068,8 @@ private Object[] createExpectedRow( @Nullable String tier, @Nullable Long currSize, @Nullable Long maxSize, - @Nullable Long isLeader + @Nullable Long isLeader, + String startTime ) { return new Object[]{ @@ -1045,7 +1081,8 @@ private Object[] createExpectedRow( tier, currSize, maxSize, - isLeader + isLeader, + startTime }; } diff --git a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap index 19121a8a13bc..8a4171af51c4 100644 --- a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap +++ b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap @@ -58,6 +58,7 @@ exports[`ServicesView renders data 1`] = ` "Current size", "Max size", "Usage", + "Start Time", "Detail", "Actions", ] @@ -196,6 +197,14 @@ exports[`ServicesView renders data 1`] = ` "show": true, "width": 140, }, + Object { + "Aggregated": [Function], + "Cell": [Function], + "Header": "Start Time", + "accessor": "start_time", + "show": true, + "width": 200, + }, Object { "Aggregated": [Function], "Cell": [Function], @@ -223,6 +232,7 @@ exports[`ServicesView renders data 1`] = ` Array [ Array [ Object { + "start_time": 0, "curr_size": 0, "host": "localhost", "is_leader": 0, @@ -234,6 +244,7 @@ exports[`ServicesView renders data 1`] = ` "tls_port": -1, }, Object { + "start_time": 0, "curr_size": 179744287, "host": "localhost", "is_leader": 0, diff --git a/web-console/src/views/services-view/services-view.spec.tsx b/web-console/src/views/services-view/services-view.spec.tsx index 21d51e9c5dd5..c7ddc86cf012 100644 --- a/web-console/src/views/services-view/services-view.spec.tsx +++ b/web-console/src/views/services-view/services-view.spec.tsx @@ -48,6 +48,7 @@ jest.mock('../../utils', () => { curr_size: 0, max_size: 0, is_leader: 0, + start_time: 0, }, { service: 'localhost:8083', @@ -63,6 +64,7 @@ jest.mock('../../utils', () => { segmentsToDrop: 0, segmentsToLoadSize: 0, segmentsToDropSize: 0, + start_time: 0, }, ], ], diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index aa2a934533e7..4b112222993f 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -67,6 +67,7 @@ const allColumns: string[] = [ 'Current size', 'Max size', 'Usage', + 'Start Time', 'Detail', ACTION_COLUMN_LABEL, ]; @@ -74,7 +75,17 @@ const allColumns: string[] = [ const tableColumns: Record = { 'full': allColumns, 'no-sql': allColumns, - 'no-proxy': ['Service', 'Type', 'Tier', 'Host', 'Port', 'Current size', 'Max size', 'Usage'], + 'no-proxy': [ + 'Service', + 'Type', + 'Tier', + 'Host', + 'Port', + 'Current size', + 'Max size', + 'Usage', + 'Start Time', + ], }; function formatQueues( @@ -127,6 +138,7 @@ interface ServiceQueryResultRow { readonly max_size: NumberLike; readonly plaintext_port: number; readonly tls_port: number; + readonly start_time: string; } interface LoadQueueStatus { @@ -180,7 +192,8 @@ export class ServicesView extends React.PureComponent '', + }, { Header: 'Detail', show: visibleColumns.shown('Detail'), From 737c77cd19c14b8995ac055c12edf708c7778c01 Mon Sep 17 00:00:00 2001 From: Atul Mohan Date: Tue, 15 Nov 2022 16:49:18 -0800 Subject: [PATCH 2/9] Spelling fixes --- web-console/src/views/services-view/services-view.tsx | 9 ++++----- website/.spelling | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index 4b112222993f..9678c9178859 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -67,7 +67,7 @@ const allColumns: string[] = [ 'Current size', 'Max size', 'Usage', - 'Start Time', + 'Start time', 'Detail', ACTION_COLUMN_LABEL, ]; @@ -84,7 +84,7 @@ const tableColumns: Record = { 'Current size', 'Max size', 'Usage', - 'Start Time', + 'Start time', ], }; @@ -224,7 +224,6 @@ ORDER BY curr_size: s.currSize, max_size: s.maxSize, tls_port: -1, - start_time: s.start_time, }; }); } @@ -525,8 +524,8 @@ ORDER BY }, }, { - Header: 'Start Time', - show: visibleColumns.shown('Start Time'), + Header: 'Start time', + show: visibleColumns.shown('Start time'), accessor: 'start_time', width: 200, Cell: this.renderFilterableCell('start_time'), diff --git a/website/.spelling b/website/.spelling index c27c4d8ccc80..a5ca640382d3 100644 --- a/website/.spelling +++ b/website/.spelling @@ -457,6 +457,7 @@ smooshed splittable ssl sslmode +start_time stdout storages stringDictionaryEncoding From f9f4415ad27581ae75983ffd82c5ebfb0b147e1a Mon Sep 17 00:00:00 2001 From: Atul Mohan Date: Wed, 16 Nov 2022 08:58:48 -0800 Subject: [PATCH 3/9] Changes to test --- .../__snapshots__/services-view.spec.tsx.snap | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap index 8a4171af51c4..4159dedc2de0 100644 --- a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap +++ b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap @@ -58,7 +58,7 @@ exports[`ServicesView renders data 1`] = ` "Current size", "Max size", "Usage", - "Start Time", + "Start time", "Detail", "Actions", ] @@ -200,7 +200,7 @@ exports[`ServicesView renders data 1`] = ` Object { "Aggregated": [Function], "Cell": [Function], - "Header": "Start Time", + "Header": "Start time", "accessor": "start_time", "show": true, "width": 200, @@ -232,7 +232,6 @@ exports[`ServicesView renders data 1`] = ` Array [ Array [ Object { - "start_time": 0, "curr_size": 0, "host": "localhost", "is_leader": 0, @@ -240,11 +239,11 @@ exports[`ServicesView renders data 1`] = ` "plaintext_port": 8082, "service": "localhost:8082", "service_type": "broker", + "start_time": 0, "tier": null, "tls_port": -1, }, Object { - "start_time": 0, "curr_size": 179744287, "host": "localhost", "is_leader": 0, @@ -256,6 +255,7 @@ exports[`ServicesView renders data 1`] = ` "segmentsToLoadSize": 0, "service": "localhost:8083", "service_type": "historical", + "start_time": 0, "tier": "_default_tier", "tls_port": -1, }, From cdc0ed7a7f2cf8e2693cedb3ab8047295228621e Mon Sep 17 00:00:00 2001 From: Atul Mohan Date: Wed, 16 Nov 2022 14:17:21 -0800 Subject: [PATCH 4/9] Fix services view for nosql --- .../src/views/services-view/services-view.tsx | 41 +++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index 9678c9178859..147b54b8c41c 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -58,23 +58,32 @@ import { BasicAction } from '../../utils/basic-action'; import './services-view.scss'; -const allColumns: string[] = [ - 'Service', - 'Type', - 'Tier', - 'Host', - 'Port', - 'Current size', - 'Max size', - 'Usage', - 'Start time', - 'Detail', - ACTION_COLUMN_LABEL, -]; - const tableColumns: Record = { - 'full': allColumns, - 'no-sql': allColumns, + 'full': [ + 'Service', + 'Type', + 'Tier', + 'Host', + 'Port', + 'Current size', + 'Max size', + 'Usage', + 'Start time', + 'Detail', + ACTION_COLUMN_LABEL, + ], + 'no-sql': [ + 'Service', + 'Type', + 'Tier', + 'Host', + 'Port', + 'Current size', + 'Max size', + 'Usage', + 'Detail', + ACTION_COLUMN_LABEL, + ], 'no-proxy': [ 'Service', 'Type', From f1c46d320c5bc390a924316cb55b354c0da2a46f Mon Sep 17 00:00:00 2001 From: Atul Mohan Date: Mon, 28 Nov 2022 09:20:42 -0800 Subject: [PATCH 5/9] Log attempt to announce diff values under same path --- .../java/org/apache/druid/curator/announcement/Announcer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java b/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java index b65d85644d40..7ec8eb589c8a 100644 --- a/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java +++ b/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java @@ -314,7 +314,7 @@ public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) th if (oldBytes == null) { created = true; } else if (!Arrays.equals(oldBytes, bytes)) { - throw new IAE("Cannot reannounce different values under the same path"); + log.error("Ignoring attempt to announce different values under same path"); } } } From 4aaa647a73d05cb628631eb075bbcfe9561f23b3 Mon Sep 17 00:00:00 2001 From: Atul Mohan Date: Mon, 28 Nov 2022 10:11:45 -0800 Subject: [PATCH 6/9] Checkstyle --- .../java/org/apache/druid/curator/announcement/Announcer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java b/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java index 7ec8eb589c8a..57619ea8faf0 100644 --- a/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java +++ b/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java @@ -29,7 +29,6 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.utils.ZKPaths; import org.apache.druid.curator.cache.PathChildrenCacheFactory; -import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.lifecycle.LifecycleStart; import org.apache.druid.java.util.common.lifecycle.LifecycleStop; From 1a9c5e945349a23536fbe1aa7df0cd09fdf4242a Mon Sep 17 00:00:00 2001 From: Atul Mohan Date: Tue, 21 Mar 2023 15:30:42 -0700 Subject: [PATCH 7/9] Default start time for int tests --- .../tests/security/AbstractAuthConfigurationTest.java | 9 +++++---- .../resources/results/auth_test_sys_schema_servers.json | 6 ++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/integration-tests/src/test/java/org/apache/druid/tests/security/AbstractAuthConfigurationTest.java b/integration-tests/src/test/java/org/apache/druid/tests/security/AbstractAuthConfigurationTest.java index 5c6e02745960..1c5867916137 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/security/AbstractAuthConfigurationTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/security/AbstractAuthConfigurationTest.java @@ -276,7 +276,7 @@ public void test_systemSchemaAccess_admin() throws Exception verifySystemSchemaServerQuery( adminClient, SYS_SCHEMA_SERVERS_QUERY, - getServersWithoutCurrentSize(adminServers) + getServersWithoutCurrentSizeAndStartTime(adminServers) ); LOG.info("Checking sys.server_segments query as admin..."); @@ -767,7 +767,7 @@ protected void verifySystemSchemaQueryBase( String content = responseHolder.getContent(); List> responseMap = jsonMapper.readValue(content, SYS_SCHEMA_RESULTS_TYPE_REFERENCE); if (isServerQuery) { - responseMap = getServersWithoutCurrentSize(responseMap); + responseMap = getServersWithoutCurrentSizeAndStartTime(responseMap); } Assert.assertEquals(responseMap, expectedResults); } @@ -914,7 +914,7 @@ protected void setExpectedSystemSchemaObjects() throws IOException SYS_SCHEMA_RESULTS_TYPE_REFERENCE ); - adminServers = getServersWithoutCurrentSize( + adminServers = getServersWithoutCurrentSizeAndStartTime( jsonMapper.readValue( fillServersTemplate( config, @@ -937,13 +937,14 @@ protected void setExpectedSystemSchemaObjects() throws IOException * curr_size on historicals changes because cluster state is not isolated across different * integration tests, zero it out for consistent test results */ - protected static List> getServersWithoutCurrentSize(List> servers) + protected static List> getServersWithoutCurrentSizeAndStartTime(List> servers) { return Lists.transform( servers, (server) -> { Map newServer = new HashMap<>(server); newServer.put("curr_size", 0); + newServer.put("start_time", "0"); return newServer; } ); diff --git a/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json b/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json index d27614e2ecef..776c2ba2f5ed 100644 --- a/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json +++ b/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json @@ -8,7 +8,8 @@ "tier": "_default_tier", "curr_size": 2208932412, "max_size": 5000000000, - "is_leader": %%NON_LEADER%% + "is_leader": %%NON_LEADER%%, + "start_time": "0" }, { "server": "%%BROKER%%:8282", @@ -19,6 +20,7 @@ "tier": "_default_tier", "curr_size": 0, "max_size": 1000000000, - "is_leader": %%NON_LEADER%% + "is_leader": %%NON_LEADER%%, + "start_time": "0" } ] From 839f2a58653fe450f8d70597369d7813155d848c Mon Sep 17 00:00:00 2001 From: Atul Mohan Date: Mon, 10 Apr 2023 16:38:12 -0700 Subject: [PATCH 8/9] Fix doc comments --- docs/querying/sql-metadata-tables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/querying/sql-metadata-tables.md b/docs/querying/sql-metadata-tables.md index b4583ddb4e43..3d6093659560 100644 --- a/docs/querying/sql-metadata-tables.md +++ b/docs/querying/sql-metadata-tables.md @@ -208,7 +208,7 @@ Servers table lists all discovered servers in the cluster. |current_size|BIGINT|Current size of segments in bytes on this server. Only valid for HISTORICAL type, for other types it's 0| |max_size|BIGINT|Max size in bytes this server recommends to assign to segments see [druid.server.maxSize](../configuration/index.md#historical-general-configuration). Only valid for HISTORICAL type, for other types it's 0| |is_leader|BIGINT|1 if the server is currently the 'leader' (for services which have the concept of leadership), otherwise 0 if the server is not the leader, or the default long value (0 or null depending on `druid.generic.useDefaultValueForNull`) if the server type does not have the concept of leadership| -|start_time|STRING|Timestamp in ISO8601 format corresponding to when the server was announced in the cluster| +|start_time|STRING|Timestamp in ISO8601 format when the server was announced in the cluster| To retrieve information about all servers, use the query: ```sql From e2ae733bbc8c949385db45642c74492b56a4089f Mon Sep 17 00:00:00 2001 From: Atul Mohan Date: Thu, 13 Apr 2023 09:44:29 -0700 Subject: [PATCH 9/9] bind announcer only in standalone mode --- .../apache/druid/curator/announcement/Announcer.java | 3 ++- .../main/java/org/apache/druid/cli/CliOverlord.java | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java b/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java index 57619ea8faf0..b65d85644d40 100644 --- a/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java +++ b/server/src/main/java/org/apache/druid/curator/announcement/Announcer.java @@ -29,6 +29,7 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.utils.ZKPaths; import org.apache.druid.curator.cache.PathChildrenCacheFactory; +import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.lifecycle.LifecycleStart; import org.apache.druid.java.util.common.lifecycle.LifecycleStop; @@ -313,7 +314,7 @@ public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) th if (oldBytes == null) { created = true; } else if (!Arrays.equals(oldBytes, bytes)) { - log.error("Ignoring attempt to announce different values under same path"); + throw new IAE("Cannot reannounce different values under the same path"); } } } diff --git a/services/src/main/java/org/apache/druid/cli/CliOverlord.java b/services/src/main/java/org/apache/druid/cli/CliOverlord.java index e4383c673c5d..8267329e2d8a 100644 --- a/services/src/main/java/org/apache/druid/cli/CliOverlord.java +++ b/services/src/main/java/org/apache/druid/cli/CliOverlord.java @@ -267,13 +267,13 @@ public void configure(Binder binder) if (standalone) { LifecycleModule.register(binder, Server.class); - } - bindAnnouncer( - binder, - IndexingService.class, - DiscoverySideEffectsProvider.create() - ); + bindAnnouncer( + binder, + IndexingService.class, + DiscoverySideEffectsProvider.create() + ); + } Jerseys.addResource(binder, SelfDiscoveryResource.class); LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class));