From 294556da1c649b7b652dddaf20841774142992f8 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sun, 27 Aug 2023 22:12:53 +0530 Subject: [PATCH 01/36] move smc to coordinator --- ...ruidSchemaInternRowSignatureBenchmark.java | 10 +- .../timeline/SegmentStatusInCluster.java | 51 ++- .../timeline/SegmentStatusInClusterTest.java | 2 +- .../apache/druid/client/BrokerServerView.java | 11 + .../druid/client/CoordinatorServerView.java | 250 +++++++++++- ...ryConfig.java => InternalQueryConfig.java} | 2 +- .../client/coordinator/CoordinatorClient.java | 6 + .../coordinator/CoordinatorClientImpl.java | 19 + .../metadata}/AvailableSegmentMetadata.java | 15 +- .../segment/metadata/DatasourceSchema.java | 32 ++ .../metadata}/SegmentMetadataCache.java | 105 +++-- .../metadata/SegmentMetadataCacheConfig.java | 46 +++ .../server/http/DataSourcesResource.java | 4 +- .../druid/server/http/MetadataResource.java | 47 ++- ...Test.java => InternalQueryConfigTest.java} | 20 +- .../server/http/DataSourcesResourceTest.java | 8 +- .../server/http/MetadataResourceTest.java | 8 +- .../java/org/apache/druid/cli/CliBroker.java | 6 +- .../org/apache/druid/cli/CliCoordinator.java | 45 +++ .../calcite/planner/CalcitePlannerModule.java | 3 +- .../planner/SegmentMetadataCacheConfig.java | 100 ----- .../schema/BrokerSegmentMetadataCache.java | 75 ++++ .../BrokerSegmentMetadataCacheConfig.java | 31 ++ .../schema/BrokerSegmentMetadataView.java | 360 ++++++++++++++++++ .../schema/DruidCalciteSchemaModule.java | 23 +- .../druid/sql/calcite/schema/DruidSchema.java | 20 +- .../calcite/schema/MetadataSegmentView.java | 242 ------------ .../PhysicalDatasourceMetadataBuilder.java | 42 ++ .../sql/calcite/schema/SystemSchema.java | 54 ++- .../org/apache/druid/sql/guice/SqlModule.java | 2 +- .../SegmentMetadataCacheConfigTest.java | 3 +- .../schema/DruidSchemaNoDataInitTest.java | 7 +- .../SegmentDataCacheConcurrencyTest.java | 9 +- .../schema/SegmentMetadataCacheCommon.java | 2 +- .../schema/SegmentMetadataCacheTest.java | 38 +- .../sql/calcite/schema/SystemSchemaTest.java | 23 +- .../druid/sql/calcite/util/CalciteTests.java | 6 +- .../sql/calcite/util/QueryFrameworkUtils.java | 11 +- 38 files changed, 1225 insertions(+), 513 deletions(-) rename server/src/main/java/org/apache/druid/client/{BrokerInternalQueryConfig.java => InternalQueryConfig.java} (97%) rename {sql/src/main/java/org/apache/druid/sql/calcite/schema => server/src/main/java/org/apache/druid/segment/metadata}/AvailableSegmentMetadata.java (90%) create mode 100644 server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java rename {sql/src/main/java/org/apache/druid/sql/calcite/schema => server/src/main/java/org/apache/druid/segment/metadata}/SegmentMetadataCache.java (92%) create mode 100644 server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java rename server/src/test/java/org/apache/druid/client/{BrokerInternalQueryConfigTest.java => InternalQueryConfigTest.java} (84%) delete mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfig.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java delete mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 138509a1a80e..e87efb348079 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -22,7 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.guava.Sequence; @@ -38,8 +38,8 @@ import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.planner.PlannerConfig; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; -import org.apache.druid.sql.calcite.schema.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.partition.LinearShardSpec; @@ -80,7 +80,7 @@ public SegmentMetadataCacheForBenchmark( final JoinableFactory joinableFactory, final PlannerConfig config, final Escalator escalator, - final BrokerInternalQueryConfig brokerInternalQueryConfig + final InternalQueryConfig internalQueryConfig ) { super( @@ -90,7 +90,7 @@ public SegmentMetadataCacheForBenchmark( joinableFactory, SegmentMetadataCacheConfig.create(), escalator, - brokerInternalQueryConfig, + internalQueryConfig, new NoopServiceEmitter() ); } diff --git a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java index 4e5577f76039..26d89d79ed34 100644 --- a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java +++ b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java @@ -53,25 +53,43 @@ public class SegmentStatusInCluster implements Comparable callback.segmentAdded(server, segment)); + } + + private void serverAddedSegment2(final DruidServerMetadata server, final DataSegment segment) + { + final SegmentId segmentId = segment.getId(); + synchronized (lock) { + // in theory we could probably just filter this to ensure we don't put ourselves in here, to make broker tree + // query topologies, but for now just skip all brokers, so we don't create some sort of wild infinite query + // loop... + if (!server.getType().equals(ServerType.BROKER)) { + log.debug("Adding segment[%s] for server[%s]", segment, server); + ServerSelector selector = selectors.get(segmentId); + if (selector == null) { + selector = new ServerSelector(segment, tierSelectorStrategy); + + VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); + if (timeline == null) { + // broker needs to skip tombstones + timeline = new VersionedIntervalTimeline<>(Ordering.natural(), true); + timelines2.put(segment.getDataSource(), timeline); + } + + timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); + selectors.put(segmentId, selector); + } + + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); + if (inventoryValue == null) { + log.warn( + "Could not find server[%s] in inventory. Skipping addition of segment[%s].", + server.getName(), + segmentId + ); + return; + } else { + queryableDruidServer = addServer(inventoryValue); + } + } + selector.addServerAndUpdateSegment(queryableDruidServer, segment); + } + // run the callbacks, even if the segment came from a broker, lets downstream watchers decide what to do with it + runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); + } + } + + private QueryableDruidServer addServer(DruidServer server) + { + QueryableDruidServer retVal = new QueryableDruidServer<>(server, makeDirectClient(server)); + QueryableDruidServer exists = clients.put(server.getName(), retVal); + if (exists != null) { + log.warn("QueryRunner for server[%s] already exists!? Well it's getting replaced", server); + } + + return retVal; + } + + private DirectDruidClient makeDirectClient(DruidServer server) + { + return new DirectDruidClient( + warehouse, + queryWatcher, + smileMapper, + httpClient, + server.getScheme(), + server.getHost(), + emitter + ); } private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) @@ -178,7 +296,9 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen log.warn("Told to remove non-existant segment[%s]", segmentId); return; } - segmentLoadInfo.removeServer(server); + if (segmentLoadInfo.removeServer(server)) { + runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); + } if (segmentLoadInfo.isEmpty()) { VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); segmentLoadInfos.remove(segmentId); @@ -197,6 +317,62 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen segment.getInterval(), segment.getVersion() ); + } else { + runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); + } + } + } + } + + private void serverRemovedSegment2(DruidServerMetadata server, DataSegment segment) + { + final SegmentId segmentId = segment.getId(); + final ServerSelector selector; + + synchronized (lock) { + log.debug("Removing segment[%s] from server[%s].", segmentId, server); + + // we don't store broker segments here, but still run the callbacks for the segment being removed from the server + // since the broker segments are not stored on the timeline, do not fire segmentRemoved event + selector = selectors.get(segmentId); + if (selector == null) { + log.warn("Told to remove non-existant segment[%s]", segmentId); + return; + } + + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + log.warn( + "Could not find server[%s] in inventory. Skipping removal of segment[%s].", + server.getName(), + segmentId + ); + } else if (!selector.removeServer(queryableDruidServer)) { + log.warn( + "Asked to disassociate non-existant association between server[%s] and segment[%s]", + server, + segmentId + ); + } else { + // runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); + } + + if (selector.isEmpty()) { + VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); + selectors.remove(segmentId); + + final PartitionChunk removedPartition = timeline.remove( + segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector) + ); + + if (removedPartition == null) { + log.warn( + "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", + segment.getInterval(), + segment.getVersion() + ); + } else { + // runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); } } } @@ -210,9 +386,9 @@ public VersionedIntervalTimeline getTimeline(DataSource } } - public Map getSegmentLoadInfos() + public Set getLoadedSegmentIds() { - return segmentLoadInfos; + return segmentLoadInfos.keySet(); } @Override @@ -238,4 +414,68 @@ public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) { return baseView.isSegmentLoadedByServer(serverKey, segment); } + + private void runTimelineCallbacks(final Function function) + { + for (Map.Entry entry : timelineCallbacks.entrySet()) { + entry.getValue().execute( + () -> { + if (CallbackAction.UNREGISTER == function.apply(entry.getKey())) { + timelineCallbacks.remove(entry.getKey()); + } + } + ); + } + } + + @Override + public void registerTimelineCallback(Executor exec, TimelineCallback callback) + { + timelineCallbacks.put(callback, exec); + } + + @Override + public void registerServerRemovedCallback(Executor exec, ServerRemovedCallback callback) + { + throw new UnsupportedOperationException(); + } + + @Override + public void registerSegmentCallback(Executor exec, SegmentCallback callback) + { + baseView.registerSegmentCallback(exec, callback); + } + + @Override + public Optional> getTimeline(DataSourceAnalysis analysis) + { + final TableDataSource table = + analysis.getBaseTableDataSource() + .orElseThrow(() -> new ISE("Cannot handle base datasource: %s", analysis.getBaseDataSource())); + + synchronized (lock) { + return Optional.ofNullable(timelines2.get(table.getName())); + } + } + + @Override + public List getDruidServers() + { + return clients.values().stream() + .map(queryableDruidServer -> queryableDruidServer.getServer().toImmutableDruidServer()) + .collect(Collectors.toList()); + } + + @Override + public QueryRunner getQueryRunner(DruidServer server) + { + synchronized (lock) { + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + log.error("No QueryRunner found for server name[%s].", server.getName()); + return null; + } + return queryableDruidServer.getQueryRunner(); + } + } } diff --git a/server/src/main/java/org/apache/druid/client/BrokerInternalQueryConfig.java b/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java similarity index 97% rename from server/src/main/java/org/apache/druid/client/BrokerInternalQueryConfig.java rename to server/src/main/java/org/apache/druid/client/InternalQueryConfig.java index 9b893778d60b..3c1d4d96022b 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerInternalQueryConfig.java +++ b/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java @@ -29,7 +29,7 @@ * should add to their query payload. The runtime properties for this class * have the prefix "druid.broker.internal.query.config." */ -public class BrokerInternalQueryConfig +public class InternalQueryConfig { @JsonProperty private Map context = new HashMap<>(); diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index c497dcb68940..812f35d4622e 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -20,12 +20,16 @@ package org.apache.druid.client.coordinator; import com.google.common.util.concurrent.ListenableFuture; +import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentStatusInCluster; import org.joda.time.Interval; +import java.util.Iterator; import java.util.List; +import java.util.Set; public interface CoordinatorClient { @@ -44,6 +48,8 @@ public interface CoordinatorClient */ ListenableFuture> fetchUsedSegments(String dataSource, List intervals); + ListenableFuture> fetchDatasourceSchema(List datasources); + /** * Returns a new instance backed by a ServiceClient which follows the provided retryPolicy */ diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index e023d9827bd4..dae00c73bea0 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -26,15 +26,19 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.http.client.response.BytesFullResponseHandler; +import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.RequestBuilder; import org.apache.druid.rpc.ServiceClient; import org.apache.druid.rpc.ServiceRetryPolicy; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentStatusInCluster; import org.jboss.netty.handler.codec.http.HttpMethod; import org.joda.time.Interval; +import java.util.Iterator; import java.util.List; +import java.util.Set; public class CoordinatorClientImpl implements CoordinatorClient { @@ -106,6 +110,21 @@ public ListenableFuture> fetchUsedSegments(String dataSource, ); } + @Override + public ListenableFuture> fetchDatasourceSchema(List datasources) + { + final String path = "/druid/coordinator/v1/metadata/datasourceSchema"; + + return FutureUtils.transform( + client.asyncRequest( + new RequestBuilder(HttpMethod.POST, path) + .jsonContent(jsonMapper, datasources), + new BytesFullResponseHandler() + ), + holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), new TypeReference>() {}) + ); + } + @Override public CoordinatorClientImpl withRetryPolicy(ServiceRetryPolicy retryPolicy) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/AvailableSegmentMetadata.java b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java similarity index 90% rename from sql/src/main/java/org/apache/druid/sql/calcite/schema/AvailableSegmentMetadata.java rename to server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java index 20d9b7e0ad9d..104194531769 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/AvailableSegmentMetadata.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.schema; +package org.apache.druid.segment.metadata; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.coordination.DruidServerMetadata; @@ -27,7 +27,7 @@ import java.util.Set; /** - * Immutable representation of RowSignature and other segment attributes needed by {@link SystemSchema.SegmentsTable} + * Immutable representation of RowSignature and other segment attributes needed by {@code SystemSchema.SegmentsTable} * This class contains the metadata of segments announced by historicals or ingestion tasks. */ public class AvailableSegmentMetadata @@ -159,4 +159,15 @@ public AvailableSegmentMetadata build() } } + @Override + public String toString() + { + return "AvailableSegmentMetadata{" + + "segment=" + segment + + ", isRealtime=" + isRealtime + + ", segmentServers=" + segmentServers + + ", numRows=" + numRows + + ", rowSignature=" + rowSignature + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java b/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java new file mode 100644 index 000000000000..cad5cb882c62 --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java @@ -0,0 +1,32 @@ +package org.apache.druid.segment.metadata; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.column.RowSignature; + +public class DatasourceSchema +{ + private final String datasource; + private final RowSignature rowSignature; + + @JsonCreator + public DatasourceSchema( + @JsonProperty("datasource") String datasource, + @JsonProperty("rowSignature") RowSignature rowSignature) + { + this.datasource = datasource; + this.rowSignature = rowSignature; + } + + @JsonProperty + public String getDatasource() + { + return datasource; + } + + @JsonProperty + public RowSignature getRowSignature() + { + return rowSignature; + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java similarity index 92% rename from sql/src/main/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCache.java rename to server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 37ff3bda313b..971c45cd3885 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.schema; +package org.apache.druid.segment.metadata; import com.fasterxml.jackson.annotation.JsonCreator; import com.google.common.annotations.VisibleForTesting; @@ -25,6 +25,7 @@ import com.google.common.base.Predicates; import com.google.common.base.Stopwatch; import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Interner; import com.google.common.collect.Interners; @@ -33,7 +34,7 @@ import com.google.common.collect.Sets; import com.google.errorprone.annotations.concurrent.GuardedBy; import com.google.inject.Inject; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ServerView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.guice.ManageLifecycle; @@ -50,7 +51,6 @@ import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.query.DruidMetrics; -import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; @@ -61,15 +61,11 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.Types; -import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.SegmentManager; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Escalator; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; -import org.apache.druid.sql.calcite.table.DatasourceTable; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; @@ -118,8 +114,7 @@ public class SegmentMetadataCache private final SegmentMetadataCacheConfig config; // Escalator, so we can attach an authentication result to queries we generate. private final Escalator escalator; - private final SegmentManager segmentManager; - private final JoinableFactory joinableFactory; + private final ExecutorService cacheExec; private final ExecutorService callbackExec; private final ServiceEmitter emitter; @@ -129,7 +124,7 @@ public class SegmentMetadataCache * Map of DataSource -> DruidTable. * This map can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final ConcurrentMap tables = new ConcurrentHashMap<>(); /** * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. @@ -186,7 +181,7 @@ public class SegmentMetadataCache * Currently, there are 2 threads that can access these variables. * * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. - * - {@link #cacheExec} periodically refreshes segment metadata and {@link DatasourceTable} if necessary + * - {@link #cacheExec} periodically refreshes segment metadata and {@link DatasourceSchema} if necessary * based on the information collected via timeline callbacks. */ private final Object lock = new Object(); @@ -204,7 +199,7 @@ public class SegmentMetadataCache private final TreeSet segmentsNeedingRefresh = new TreeSet<>(SEGMENT_ORDER); // Configured context to attach to internally generated queries. - private final BrokerInternalQueryConfig brokerInternalQueryConfig; + private final InternalQueryConfig internalQueryConfig; @GuardedBy("lock") private boolean refreshImmediately = false; @@ -223,24 +218,20 @@ public class SegmentMetadataCache public SegmentMetadataCache( final QueryLifecycleFactory queryLifecycleFactory, final TimelineServerView serverView, - final SegmentManager segmentManager, - final JoinableFactory joinableFactory, final SegmentMetadataCacheConfig config, final Escalator escalator, - final BrokerInternalQueryConfig brokerInternalQueryConfig, + final InternalQueryConfig internalQueryConfig, final ServiceEmitter emitter ) { this.queryLifecycleFactory = Preconditions.checkNotNull(queryLifecycleFactory, "queryLifecycleFactory"); Preconditions.checkNotNull(serverView, "serverView"); - this.segmentManager = segmentManager; - this.joinableFactory = joinableFactory; this.config = Preconditions.checkNotNull(config, "config"); this.columnTypeMergePolicy = config.getMetadataColumnTypeMergePolicy(); this.cacheExec = Execs.singleThreaded("DruidSchema-Cache-%d"); this.callbackExec = Execs.singleThreaded("DruidSchema-Callback-%d"); this.escalator = escalator; - this.brokerInternalQueryConfig = brokerInternalQueryConfig; + this.internalQueryConfig = internalQueryConfig; this.emitter = emitter; initServerViewTimelineCallback(serverView); @@ -421,19 +412,23 @@ void refresh(final Set segmentsToRefresh, final Set dataSourc // Rebuild the dataSources. for (String dataSource : dataSourcesToRebuild) { - final DatasourceTable.PhysicalDatasourceMetadata druidTable = buildDruidTable(dataSource); - if (druidTable == null) { - log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tables.remove(dataSource); - continue; - } - final DatasourceTable.PhysicalDatasourceMetadata oldTable = tables.put(dataSource, druidTable); - final String description = druidTable.dataSource().isGlobal() ? "global dataSource" : "dataSource"; - if (oldTable == null || !oldTable.rowSignature().equals(druidTable.rowSignature())) { - log.info("%s [%s] has new signature: %s.", description, dataSource, druidTable.rowSignature()); - } else { - log.debug("%s [%s] signature is unchanged.", description, dataSource); - } + rebuildDatasource(dataSource); + } + } + + public void rebuildDatasource(String dataSource) + { + final DatasourceSchema druidTable = buildDruidTable(dataSource); + if (druidTable == null) { + log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); + tables.remove(dataSource); + return; + } + final DatasourceSchema oldTable = tables.put(dataSource, druidTable); + if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { + log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); + } else { + log.info("[%s] signature is unchanged.", dataSource); } } @@ -449,12 +444,17 @@ public void awaitInitialization() throws InterruptedException initialized.await(); } - protected DatasourceTable.PhysicalDatasourceMetadata getDatasource(String name) + public DatasourceSchema getDatasource(String name) { return tables.get(name); } - protected Set getDatasourceNames() + public Map getDatasourceSchemaMap() + { + return ImmutableMap.copyOf(tables); + } + + public Set getDatasourceNames() { return tables.keySet(); } @@ -717,7 +717,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin throw new ISE("'segments' must all match 'dataSource'!"); } - log.debug("Refreshing metadata for dataSource[%s].", dataSource); + log.info("Refreshing metadata for dataSource[%s].", dataSource); final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, dataSource); @@ -743,7 +743,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin log.warn("Got analysis for segment [%s] we didn't ask for, ignoring.", analysis.getId()); } else { final RowSignature rowSignature = analysisToRowSignature(analysis); - log.debug("Segment[%s] has signature[%s].", segmentId, rowSignature); + log.info("Segment[%s] has signature[%s].", segmentId, rowSignature); segmentMetadataInfo.compute( dataSource, (datasourceKey, dataSourceSegments) -> { @@ -795,7 +795,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin emitter.emit(builder.build("metadatacache/refresh/time", refreshDurationMillis)); - log.debug( + log.info( "Refreshed metadata for dataSource [%s] in %,d ms (%d segments queried, %d segments left).", dataSource, refreshDurationMillis, @@ -808,7 +808,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin @VisibleForTesting @Nullable - DatasourceTable.PhysicalDatasourceMetadata buildDruidTable(final String dataSource) + public DatasourceSchema buildDruidTable(final String dataSource) { ConcurrentSkipListMap segmentsMap = segmentMetadataInfo.get(dataSource); @@ -836,26 +836,10 @@ DatasourceTable.PhysicalDatasourceMetadata buildDruidTable(final String dataSour final RowSignature.Builder builder = RowSignature.builder(); columnTypes.forEach(builder::add); - final TableDataSource tableDataSource; - - // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing - // in the segment cache, which in this case belongs to the broker meaning only broadcast segments live here) - // to be joinable, it must be possibly joinable according to the factory. we only consider broadcast datasources - // at this time, and isGlobal is currently strongly coupled with joinable, so only make a global table datasource - // if also joinable - final GlobalTableDataSource maybeGlobal = new GlobalTableDataSource(dataSource); - final boolean isJoinable = joinableFactory.isDirectlyJoinable(maybeGlobal); - final boolean isBroadcast = segmentManager.getDataSourceNames().contains(dataSource); - if (isBroadcast && isJoinable) { - tableDataSource = maybeGlobal; - } else { - tableDataSource = new TableDataSource(dataSource); - } - return new DatasourceTable.PhysicalDatasourceMetadata(tableDataSource, builder.build(), isJoinable, isBroadcast); + return new DatasourceSchema(dataSource, builder.build()); } - @VisibleForTesting - Map getSegmentMetadataSnapshot() + public Map getSegmentMetadataSnapshot() { final Map segmentMetadata = Maps.newHashMapWithExpectedSize(totalSegments); for (ConcurrentSkipListMap val : segmentMetadataInfo.values()) { @@ -864,6 +848,15 @@ Map getSegmentMetadataSnapshot() return segmentMetadata; } + @Nullable + public AvailableSegmentMetadata getAvailableSegmentMetadata(String datasource, SegmentId segmentId) + { + if (!segmentMetadataInfo.containsKey(datasource)) { + return null; + } + return segmentMetadataInfo.get(datasource).get(segmentId); + } + /** * Returns total number of segments. This method doesn't use the lock intentionally to avoid expensive contention. * As a result, the returned value might be inexact. @@ -926,7 +919,7 @@ protected Sequence runSegmentMetadataQuery( false, // disable the parallel merge because we don't care about the merge and don't want to consume its resources QueryContexts.override( - brokerInternalQueryConfig.getContext(), + internalQueryConfig.getContext(), QueryContexts.BROKER_PARALLEL_MERGE_KEY, false ), diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java new file mode 100644 index 000000000000..c59ee23a77b2 --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -0,0 +1,46 @@ +package org.apache.druid.segment.metadata; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.Period; + +public class SegmentMetadataCacheConfig +{ + @JsonProperty + private boolean awaitInitializationOnStart = true; + + @JsonProperty + private Period metadataRefreshPeriod = new Period("PT1M"); + + @JsonProperty + private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = + new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); + + public static SegmentMetadataCacheConfig create() + { + return new SegmentMetadataCacheConfig(); + } + + public static SegmentMetadataCacheConfig create( + String metadataRefreshPeriod + ) + { + SegmentMetadataCacheConfig config = new SegmentMetadataCacheConfig(); + config.metadataRefreshPeriod = new Period(metadataRefreshPeriod); + return config; + } + + public boolean isAwaitInitializationOnStart() + { + return awaitInitializationOnStart; + } + + public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() + { + return metadataColumnTypeMergePolicy; + } + + public Period getMetadataRefreshPeriod() + { + return metadataRefreshPeriod; + } +} diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 1e146736da13..9fa73970100f 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -487,13 +487,13 @@ public Response getDatasourceLoadstatus( private SegmentsLoadStatistics computeSegmentLoadStatistics(Iterable segments) { - Map segmentLoadInfos = serverInventoryView.getSegmentLoadInfos(); + Set loadedSegmentIds = serverInventoryView.getLoadedSegmentIds(); int numPublishedSegments = 0; int numUnavailableSegments = 0; int numLoadedSegments = 0; for (DataSegment segment : segments) { numPublishedSegments++; - if (!segmentLoadInfos.containsKey(segment.getId())) { + if (!loadedSegmentIds.contains(segment.getId())) { numUnavailableSegments++; } else { numLoadedSegments++; diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 4f9631c1cfd2..9c9622e05e50 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -28,6 +28,9 @@ import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.Segments; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.server.JettyUtils; import org.apache.druid.server.coordinator.DruidCoordinator; @@ -54,7 +57,9 @@ import javax.ws.rs.core.UriInfo; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; @@ -69,19 +74,22 @@ public class MetadataResource private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; private final DruidCoordinator coordinator; + private final SegmentMetadataCache segmentMetadataCache; @Inject public MetadataResource( SegmentsMetadataManager segmentsMetadataManager, IndexerMetadataStorageCoordinator metadataStorageCoordinator, AuthorizerMapper authorizerMapper, - DruidCoordinator coordinator + DruidCoordinator coordinator, + SegmentMetadataCache segmentMetadataCache ) { this.segmentsMetadataManager = segmentsMetadataManager; this.metadataStorageCoordinator = metadataStorageCoordinator; this.authorizerMapper = authorizerMapper; this.coordinator = coordinator; + this.segmentMetadataCache = segmentMetadataCache; } @GET @@ -185,21 +193,40 @@ private Response getAllUsedSegmentsWithAdditionalDetails( .flatMap(t -> t.getSegments().stream()); final Set overshadowedSegments = dataSourcesSnapshot.getOvershadowedSegments(); + Set segmentAlreadySeen = new HashSet<>(); final Stream segmentStatus = usedSegments.map(segment -> { // The replication factor for unloaded segments is 0 as they will be unloaded soon boolean isOvershadowed = overshadowedSegments.contains(segment); + AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata(segment.getDataSource(), segment.getId()); Integer replicationFactor = isOvershadowed ? (Integer) 0 : coordinator.getReplicationFactor(segment.getId()); - - return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor); + Long numRows = (null != availableSegmentMetadata) ? availableSegmentMetadata.getNumRows() : null; + segmentAlreadySeen.add(segment.getId()); + return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, numRows, 0L, true); }); + final Stream realtimeSegmentStatus = segmentMetadataCache + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment().getId())) + .map(availableSegmentMetadata -> { + return new SegmentStatusInCluster( + availableSegmentMetadata.getSegment(), + false, + (int) availableSegmentMetadata.getNumReplicas(), + availableSegmentMetadata.getNumRows(), + availableSegmentMetadata.isRealtime(), + false + ); + }); + final Function> raGenerator = segment -> Collections .singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getDataSegment().getDataSource())); final Iterable authorizedSegments = AuthorizationUtils.filterAuthorizedResources( req, - segmentStatus::iterator, + Stream.concat(segmentStatus, realtimeSegmentStatus)::iterator, raGenerator, authorizerMapper ); @@ -302,4 +329,16 @@ public Response getUsedSegment( } return Response.status(Response.Status.NOT_FOUND).build(); } + + @POST + @Path("/datasourceSchema") + public Response getDatasourceSchema( + List datasources + ) + { + Map datasourceSchemaMap = segmentMetadataCache.getDatasourceSchemaMap(); + datasourceSchemaMap.keySet().retainAll(datasources); + + return Response.status(Response.Status.OK).entity(datasourceSchemaMap.values()).build(); + } } diff --git a/server/src/test/java/org/apache/druid/client/BrokerInternalQueryConfigTest.java b/server/src/test/java/org/apache/druid/client/InternalQueryConfigTest.java similarity index 84% rename from server/src/test/java/org/apache/druid/client/BrokerInternalQueryConfigTest.java rename to server/src/test/java/org/apache/druid/client/InternalQueryConfigTest.java index 24b61bb1ac4b..1030947e78dd 100644 --- a/server/src/test/java/org/apache/druid/client/BrokerInternalQueryConfigTest.java +++ b/server/src/test/java/org/apache/druid/client/InternalQueryConfigTest.java @@ -39,7 +39,7 @@ import java.util.HashMap; import java.util.Map; -public class BrokerInternalQueryConfigTest +public class InternalQueryConfigTest { private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); @@ -49,11 +49,11 @@ public void testSerde() throws Exception //defaults String json = "{}"; - BrokerInternalQueryConfig config = MAPPER.readValue( + InternalQueryConfig config = MAPPER.readValue( MAPPER.writeValueAsString( - MAPPER.readValue(json, BrokerInternalQueryConfig.class) + MAPPER.readValue(json, InternalQueryConfig.class) ), - BrokerInternalQueryConfig.class + InternalQueryConfig.class ); Assert.assertEquals(ImmutableMap.of(), config.getContext()); @@ -63,9 +63,9 @@ public void testSerde() throws Exception config = MAPPER.readValue( MAPPER.writeValueAsString( - MAPPER.readValue(json, BrokerInternalQueryConfig.class) + MAPPER.readValue(json, InternalQueryConfig.class) ), - BrokerInternalQueryConfig.class + InternalQueryConfig.class ); Map expected = new HashMap<>(); @@ -84,9 +84,9 @@ public void testMalfomattedContext() throws Exception String malformedJson = "{\"priority: 5}"; MAPPER.readValue( MAPPER.writeValueAsString( - MAPPER.readValue(malformedJson, BrokerInternalQueryConfig.class) + MAPPER.readValue(malformedJson, InternalQueryConfig.class) ), - BrokerInternalQueryConfig.class + InternalQueryConfig.class ); } @@ -104,7 +104,7 @@ public void configure(Binder binder) { binder.install(new ConfigModule()); binder.install(new DruidGuiceExtensions()); - JsonConfigProvider.bind(binder, "druid.broker.internal.query.config", BrokerInternalQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.internal.query.config", InternalQueryConfig.class); } @Provides @@ -115,7 +115,7 @@ public ObjectMapper jsonMapper() } } ); - BrokerInternalQueryConfig config = injector.getInstance(BrokerInternalQueryConfig.class); + InternalQueryConfig config = injector.getInstance(InternalQueryConfig.class); Assert.assertEquals(ImmutableMap.of(), config.getContext()); } } diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 551151458546..94b9bf8a84d0 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -1302,7 +1302,7 @@ public void testGetDatasourceLoadstatusDefault() // Test when datasource fully loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(completedLoadInfoMap).once(); + EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1318,7 +1318,7 @@ public void testGetDatasourceLoadstatusDefault() // Test when datasource half loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(halfLoadedInfoMap).once(); + EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1381,7 +1381,7 @@ public void testGetDatasourceLoadstatusSimple() // Test when datasource fully loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(completedLoadInfoMap).once(); + EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1397,7 +1397,7 @@ public void testGetDatasourceLoadstatusSimple() // Test when datasource half loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(halfLoadedInfoMap).once(); + EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index d5ab6aaaa1a5..717db5f03caa 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -112,11 +112,11 @@ public void testGetAllSegmentsWithOvershadowedStatus() final List resultList = extractSegmentStatusList(response); Assert.assertEquals(resultList.size(), 4); - Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2), resultList.get(0)); - Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null), resultList.get(1)); - Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1), resultList.get(2)); + Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 5L, 0L, true), resultList.get(0)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 5L, 0L, true), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, 5L, 0L, true), resultList.get(2)); // Replication factor should be 0 as the segment is overshadowed - Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, 5L, 0L, true), resultList.get(3)); } @Test diff --git a/services/src/main/java/org/apache/druid/cli/CliBroker.java b/services/src/main/java/org/apache/druid/cli/CliBroker.java index dd9c3c3c0b59..849444c8ec83 100644 --- a/services/src/main/java/org/apache/druid/cli/CliBroker.java +++ b/services/src/main/java/org/apache/druid/cli/CliBroker.java @@ -26,7 +26,7 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.name.Names; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.BrokerServerView; import org.apache.druid.client.CachingClusteredClient; @@ -70,6 +70,7 @@ import org.apache.druid.server.initialization.jetty.JettyServerInitializer; import org.apache.druid.server.metrics.QueryCountStatsProvider; import org.apache.druid.server.router.TieredBrokerConfig; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataView; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.PruneLoadSpec; import org.eclipse.jetty.server.Server; @@ -127,6 +128,7 @@ protected List getModules() binder.bind(CachingClusteredClient.class).in(LazySingleton.class); LifecycleModule.register(binder, BrokerServerView.class); + LifecycleModule.register(binder, BrokerSegmentMetadataView.class); binder.bind(TimelineServerView.class).to(BrokerServerView.class).in(LazySingleton.class); JsonConfigProvider.bind(binder, "druid.broker.cache", CacheConfig.class); @@ -137,7 +139,7 @@ protected List getModules() JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); JsonConfigProvider.bind(binder, "druid.broker.retryPolicy", RetryQueryRunnerConfig.class); JsonConfigProvider.bind(binder, "druid.broker.segment", BrokerSegmentWatcherConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.internal.query.config", BrokerInternalQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.internal.query.config", InternalQueryConfig.class); binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 5327f0b3a764..93e7dad644e4 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -37,19 +37,33 @@ import com.google.inject.util.Providers; import org.apache.curator.framework.CuratorFramework; import org.apache.druid.audit.AuditManager; +import org.apache.druid.client.BrokerSegmentWatcherConfig; +import org.apache.druid.client.CachingClusteredClient; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.HttpServerInventoryViewResource; +import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig; +import org.apache.druid.client.selector.ServerSelectorStrategy; +import org.apache.druid.client.selector.TierSelectorStrategy; import org.apache.druid.discovery.NodeRole; +import org.apache.druid.guice.BrokerProcessingModule; import org.apache.druid.guice.ConditionalMultibind; import org.apache.druid.guice.ConfigProvider; +import org.apache.druid.guice.DruidProcessingModule; import org.apache.druid.guice.Jerseys; +import org.apache.druid.guice.JoinableFactoryModule; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.JsonConfigurator; import org.apache.druid.guice.LazySingleton; +import org.apache.druid.guice.LegacyBrokerParallelMergeConfigModule; import org.apache.druid.guice.LifecycleModule; import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.guice.QueryRunnerFactoryModule; +import org.apache.druid.guice.QueryableModule; +import org.apache.druid.guice.SegmentWranglerModule; import org.apache.druid.guice.annotations.CoordinatorIndexingServiceDuty; import org.apache.druid.guice.annotations.CoordinatorMetadataStoreManagementDuty; import org.apache.druid.guice.annotations.EscalatedGlobal; @@ -73,8 +87,13 @@ import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.metadata.SegmentsMetadataManagerConfig; import org.apache.druid.metadata.SegmentsMetadataManagerProvider; +import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.lookup.LookupSerdeModule; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; +import org.apache.druid.server.ClientQuerySegmentWalker; import org.apache.druid.server.audit.AuditManagerProvider; import org.apache.druid.server.coordinator.DruidCoordinator; import org.apache.druid.server.coordinator.DruidCoordinatorConfig; @@ -139,6 +158,7 @@ public class CliCoordinator extends ServerRunnable { private static final Logger log = new Logger(CliCoordinator.class); private static final String AS_OVERLORD_PROPERTY = "druid.coordinator.asOverlord.enabled"; + private static final String SEGMENT_METADATA_CACHE_ENABLED = "druid.coordinator.segmentMetadataCache.enabled"; private Properties properties; private boolean beOverlord; @@ -173,6 +193,12 @@ protected List getModules() List modules = new ArrayList<>(); modules.add(JettyHttpClientModule.global()); + modules.add(new LegacyBrokerParallelMergeConfigModule()); + modules.add(new QueryRunnerFactoryModule()); + modules.add(new SegmentWranglerModule()); + modules.add(new QueryableModule()); + modules.add(new BrokerProcessingModule()); + modules.add(new JoinableFactoryModule()); modules.add( new Module() @@ -190,6 +216,8 @@ public void configure(Binder binder) binder.bind(MetadataStorage.class).toProvider(MetadataStorageProvider.class); + binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); + JsonConfigProvider.bind(binder, SegmentsMetadataManagerConfig.CONFIG_PREFIX, SegmentsMetadataManagerConfig.class); JsonConfigProvider.bind(binder, "druid.manager.rules", MetadataRuleManagerConfig.class); JsonConfigProvider.bind(binder, "druid.manager.lookups", LookupCoordinatorManagerConfig.class); @@ -200,6 +228,13 @@ public void configure(Binder binder) "druid.coordinator.balancer.cachingCost", CachingCostBalancerStrategyConfig.class ); + JsonConfigProvider.bind(binder, "druid.coordinator.segment", SegmentMetadataCacheConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.broker.retryPolicy", RetryQueryRunnerConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.segment", BrokerSegmentWatcherConfig.class); binder.bind(RedirectFilter.class).in(LazySingleton.class); if (beOverlord) { @@ -221,13 +256,17 @@ public void configure(Binder binder) .in(ManageLifecycle.class); binder.bind(LookupCoordinatorManager.class).in(LazySingleton.class); + binder.bind(CachingClusteredClient.class).in(LazySingleton.class); binder.bind(CoordinatorServerView.class); + binder.bind(TimelineServerView.class).to(CoordinatorServerView.class).in(LazySingleton.class); binder.bind(DruidCoordinator.class); LifecycleModule.register(binder, CoordinatorServerView.class); LifecycleModule.register(binder, MetadataStorage.class); LifecycleModule.register(binder, DruidCoordinator.class); + LifecycleModule.register(binder, SegmentMetadataCache.class); + binder.bind(JettyServerInitializer.class) .to(CoordinatorJettyServerInitializer.class); @@ -394,6 +433,12 @@ public static boolean isOverlord(Properties properties) return Boolean.parseBoolean(properties.getProperty(AS_OVERLORD_PROPERTY)); } + private boolean isSegmentMetadataCacheEnabled() + { + return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_ENABLED)); + } + + private static class CoordinatorCustomDutyGroupsProvider implements Provider { private Properties props; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java index 484de5c4659b..272fe5030286 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlannerModule.java @@ -25,6 +25,7 @@ import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.LazySingleton; import org.apache.druid.sql.calcite.rule.ExtensionCalciteRuleProvider; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCacheConfig; /** * The module responsible for provide bindings for the Calcite Planner. @@ -41,7 +42,7 @@ public void configure(Binder binder) // It turns out that the order of the arguments above is misleading. // We're actually binding the class to the config prefix, not the other way around. JsonConfigProvider.bind(binder, CONFIG_BASE, PlannerConfig.class); - JsonConfigProvider.bind(binder, CONFIG_BASE, SegmentMetadataCacheConfig.class); + JsonConfigProvider.bind(binder, CONFIG_BASE, BrokerSegmentMetadataCacheConfig.class); binder.bind(PlannerFactory.class).in(LazySingleton.class); binder.bind(DruidOperatorTable.class).in(LazySingleton.class); Multibinder.newSetBinder(binder, ExtensionCalciteRuleProvider.class); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfig.java deleted file mode 100644 index dc4d94b78b39..000000000000 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfig.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.sql.calcite.planner; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.sql.calcite.schema.SegmentMetadataCache; -import org.joda.time.Period; - -/** - * Configuration properties for the Broker-side cache of segment metadata - * used to infer datasources for SQL. This class shares the same config root - * as {@link PlannerConfig} to maintain backward compatibility for when - * the properties here resided in {@code PlannerConfig}. - */ -public class SegmentMetadataCacheConfig -{ - @JsonProperty - private boolean awaitInitializationOnStart = true; - - @JsonProperty - private boolean metadataSegmentCacheEnable = false; - - @JsonProperty - private long metadataSegmentPollPeriod = 60000; - - @JsonProperty - private Period metadataRefreshPeriod = new Period("PT1M"); - - @JsonProperty - private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = - new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); - - public static SegmentMetadataCacheConfig create() - { - return new SegmentMetadataCacheConfig(); - } - - public static SegmentMetadataCacheConfig create( - String metadataRefreshPeriod - ) - { - SegmentMetadataCacheConfig config = new SegmentMetadataCacheConfig(); - config.metadataRefreshPeriod = new Period(metadataRefreshPeriod); - return config; - } - - public boolean isMetadataSegmentCacheEnable() - { - return metadataSegmentCacheEnable; - } - - public Period getMetadataRefreshPeriod() - { - return metadataRefreshPeriod; - } - - public boolean isAwaitInitializationOnStart() - { - return awaitInitializationOnStart; - } - - public long getMetadataSegmentPollPeriod() - { - return metadataSegmentPollPeriod; - } - - public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() - { - return metadataColumnTypeMergePolicy; - } - - @Override - public String toString() - { - return "SegmentCacheConfig{" + - "metadataRefreshPeriod=" + metadataRefreshPeriod + - ", metadataSegmentCacheEnable=" + metadataSegmentCacheEnable + - ", metadataSegmentPollPeriod=" + metadataSegmentPollPeriod + - ", awaitInitializationOnStart=" + awaitInitializationOnStart + - ", metadataColumnTypeMergePolicy=" + metadataColumnTypeMergePolicy + - '}'; - } -} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java new file mode 100644 index 000000000000..0224841a6cbc --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -0,0 +1,75 @@ +package org.apache.druid.sql.calcite.schema; + +import com.google.inject.Inject; +import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.TimelineServerView; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; +import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.security.Escalator; +import org.apache.druid.sql.calcite.table.DatasourceTable; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * + */ +@ManageLifecycle +public class BrokerSegmentMetadataCache extends SegmentMetadataCache +{ + private static final EmittingLogger log = new EmittingLogger(SegmentMetadataCache.class); + private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; + + private final ConcurrentMap tables = new ConcurrentHashMap<>(); + + @Inject + public BrokerSegmentMetadataCache( + QueryLifecycleFactory queryLifecycleFactory, + TimelineServerView serverView, + SegmentMetadataCacheConfig config, + Escalator escalator, + InternalQueryConfig internalQueryConfig, + ServiceEmitter emitter, + PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder + ) + { + super( + queryLifecycleFactory, + serverView, + config, + escalator, + internalQueryConfig, + emitter + ); + this.physicalDatasourceMetadataBuilder = physicalDatasourceMetadataBuilder; + } + + @Override + public void rebuildDatasource(String dataSource) + { + final DatasourceSchema druidTable = buildDruidTable(dataSource); + if (druidTable == null) { + log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); + tables.remove(dataSource); + return; + } + final DatasourceTable.PhysicalDatasourceMetadata physicalDatasourceMetadata = + physicalDatasourceMetadataBuilder.build(dataSource, druidTable.getRowSignature()); + final DatasourceTable.PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); + if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { + log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); + } else { + log.info("[%s] signature is unchanged.", dataSource); + } + } + + public DatasourceTable.PhysicalDatasourceMetadata getPhysicalDatasourceMetadata(String name) + { + return tables.get(name); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java new file mode 100644 index 000000000000..d7a059710893 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -0,0 +1,31 @@ +package org.apache.druid.sql.calcite.schema; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; + +public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig +{ + @JsonProperty + private boolean metadataSegmentCacheEnable = false; + + @JsonProperty + private long metadataSegmentPollPeriod = 60000; + + @JsonProperty + private boolean useSegmentMetadataCache = false; + + public boolean isMetadataSegmentCacheEnable() + { + return metadataSegmentCacheEnable; + } + + public long getMetadataSegmentPollPeriod() + { + return metadataSegmentPollPeriod; + } + + public boolean isUseSegmentMetadataCache() + { + return useSegmentMetadataCache; + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java new file mode 100644 index 000000000000..cf92bba0eb69 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.inject.Inject; +import org.apache.druid.client.BrokerSegmentWatcherConfig; +import org.apache.druid.client.BrokerServerView; +import org.apache.druid.client.DataSegmentInterner; +import org.apache.druid.client.JsonParserIterator; +import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.client.coordinator.CoordinatorClient; +import org.apache.druid.client.selector.ServerSelector; +import org.apache.druid.common.guava.FutureUtils; +import org.apache.druid.concurrent.LifecycleLock; +import org.apache.druid.discovery.DruidLeaderClient; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.lifecycle.LifecycleStop; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.metadata.SegmentsMetadataManager; +import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.SegmentStatusInCluster; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * This class polls the Coordinator in background to keep the latest published segments. + * Provides {@link #getSegmentMetadata()} for others to get segments in metadata store. + * + * The difference between this class and {@link SegmentsMetadataManager} is that this class resides + * in Broker's memory, while {@link SegmentsMetadataManager} resides in Coordinator's memory. In + * fact, this class polls the data from {@link SegmentsMetadataManager} object in the memory of the + * currently leading Coordinator via HTTP queries. + */ +@ManageLifecycle +public class BrokerSegmentMetadataView +{ + private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataView.class); + + private final CoordinatorClient coordinatorClient; + private final BrokerSegmentWatcherConfig segmentWatcherConfig; + private final DruidLeaderClient druidLeaderClient; + private final ObjectMapper objectMapper; + + private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; + + private final boolean isMetadataSegmentCacheEnabled; + + private final boolean useSegmentMetadataCache; + + /** + * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and + * sys.segments queries return the segments in sorted order based on segmentId. + * + * Volatile since this reference is reassigned in {@code poll()} and then read in {@code getPublishedSegments()} + * from other threads. + */ + @MonotonicNonNull + private volatile ImmutableSortedSet segmentMetadata = null; + + private volatile Map datasourceSchemaMap = null; + + private final BrokerServerView brokerServerView; + + private final BrokerSegmentMetadataCache segmentMetadataCache; + + /** + * Caches the replication factor for segment IDs. In case of coordinator restarts or leadership re-elections, + * the coordinator API returns `null` replication factor until load rules are evaluated. + * The cache can be used during these periods to continue serving the previously fetched values. + */ + private final Cache segmentIdToReplicationFactor; + private final ScheduledExecutorService scheduledExec; + private final long pollPeriodInMS; + private final LifecycleLock lifecycleLock = new LifecycleLock(); + private final CountDownLatch segmentMetadataCachePopulated = new CountDownLatch(1); + + @Inject + public BrokerSegmentMetadataView( + final @Coordinator DruidLeaderClient druidLeaderClient, + final ObjectMapper objectMapper, + final CoordinatorClient coordinatorClient, + final BrokerSegmentWatcherConfig segmentWatcherConfig, + final BrokerSegmentMetadataCacheConfig config, + final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder, + final BrokerServerView brokerServerView, + final BrokerSegmentMetadataCache segmentMetadataCache + ) + { + Preconditions.checkNotNull(config, "SegmentMetadataCacheConfig"); + this.druidLeaderClient = druidLeaderClient; + this.objectMapper = objectMapper; + this.coordinatorClient = coordinatorClient; + this.segmentWatcherConfig = segmentWatcherConfig; + this.isMetadataSegmentCacheEnabled = config.isMetadataSegmentCacheEnable(); + this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); + this.scheduledExec = Execs.scheduledSingleThreaded("MetadataSegmentView-Cache--%d"); + this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() + .expireAfterAccess(10, TimeUnit.MINUTES) + .build(); + this.physicalDatasourceMetadataBuilder = physicalDatasourceMetadataBuilder; + this.useSegmentMetadataCache = config.isUseSegmentMetadataCache(); + this.brokerServerView = brokerServerView; + this.segmentMetadataCache = segmentMetadataCache; + } + + @LifecycleStart + public void start() + { + if (!lifecycleLock.canStart()) { + throw new ISE("can't start."); + } + try { + if (isMetadataSegmentCacheEnabled || !useSegmentMetadataCache) { + scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); + } + lifecycleLock.started(); + log.info("MetadataSegmentView Started. Configs isMetadataSegmentCacheEnabled [%s], useSegmentMetadataCache [%s]", + isMetadataSegmentCacheEnabled, useSegmentMetadataCache); + } + finally { + lifecycleLock.exitStart(); + } + } + + @LifecycleStop + public void stop() + { + if (!lifecycleLock.canStop()) { + throw new ISE("can't stop."); + } + log.info("MetadataSegmentView is stopping."); + if (isMetadataSegmentCacheEnabled) { + scheduledExec.shutdown(); + } + log.info("MetadataSegmentView Stopped."); + } + + private void pollSegmentMetadata() + { + log.info("Polling segment metadata from coordinator"); + + segmentMetadata = fetchSegmentMetadata(); + segmentMetadataCachePopulated.countDown(); + } + + private ImmutableSortedSet fetchSegmentMetadata() + { + final Iterator metadataSegments = + querySegmentMetadata(segmentWatcherConfig.getWatchedDataSources()); + + final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); + while (metadataSegments.hasNext()) { + final SegmentStatusInCluster segment = metadataSegments.next(); + final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); + Integer replicationFactor = segment.getReplicationFactor(); + if (replicationFactor == null) { + replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); + } else { + segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); + } + final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( + interned, + segment.isOvershadowed(), + replicationFactor, + segment.getNumRows(), + segment.isRealtime(), + segment.isPublished() + ); + builder.add(segmentStatusInCluster); + } + return builder.build(); + } + + private void pollDatasourceSchema() + { + log.info("Polling datasource schema from coordinator."); + Set datasources = useSegmentMetadataCache ? segmentMetadataCache.getDatasourceNames() : brokerServerView.getDatasourceNames(); + + Map physicalDatasourceMetadataMap = new HashMap<>(); + + for (List partition : Iterables.partition(datasources, 10)) { + List datasourceSchemas = FutureUtils.getUnchecked(coordinatorClient.fetchDatasourceSchema( + partition), true); + + for (DatasourceSchema datasourceSchema : datasourceSchemas) { + physicalDatasourceMetadataMap.put( + datasourceSchema.getDatasource(), + physicalDatasourceMetadataBuilder.build(datasourceSchema.getDatasource(), datasourceSchema.getRowSignature())); + } + } + + this.datasourceSchemaMap = physicalDatasourceMetadataMap; + } + + ImmutableSortedSet getSegmentMetadata() + { + if (isMetadataSegmentCacheEnabled) { + Uninterruptibles.awaitUninterruptibly(segmentMetadataCachePopulated); + return segmentMetadata; + } else { + return fetchSegmentMetadata(); + } + } + + protected SystemSchema.SegmentsTable.SegmentTableView getSegmentTableView() { + ImmutableSortedSet allSegmentMetadata = getSegmentMetadata(); + + final ImmutableSortedSet.Builder publishedSegmentBuilder = ImmutableSortedSet.naturalOrder(); + + Map availableSegmentMetadataMap; + + if (useSegmentMetadataCache) { + availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); + + for (SegmentStatusInCluster segmentStatusInCluster : allSegmentMetadata) { + if (segmentStatusInCluster.isPublished()) { + publishedSegmentBuilder.add(segmentStatusInCluster); + } + } + } else { + // build available segment metadata map by combining stuff from brokerServerView and numRows from data returned from coordinator + availableSegmentMetadataMap = new HashMap<>(); + Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); + for (SegmentStatusInCluster segmentStatusInCluster : allSegmentMetadata) { + if (segmentStatusInCluster.isPublished()) { + publishedSegmentBuilder.add(segmentStatusInCluster); + } + SegmentId segmentId = segmentStatusInCluster.getDataSegment().getId(); + if (!brokerSegmentMetadata.containsKey(segmentId)) { + continue; + } + ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); + + AvailableSegmentMetadata availableSegmentMetadata = + AvailableSegmentMetadata.builder( + segmentStatusInCluster.getDataSegment(), + segmentStatusInCluster.isRealtime(), + Sets.newHashSet(serverSelector.getAllServers()), + null, + segmentStatusInCluster.getNumRows() == null ? -1 : segmentStatusInCluster.getNumRows() + ) + .build(); + + availableSegmentMetadataMap.put(segmentId, availableSegmentMetadata); + + } + } + + log.info("Logging Segment table view. availableSmMap [%s], published segments [%s]", availableSegmentMetadataMap, publishedSegmentBuilder); + return new SystemSchema.SegmentsTable.SegmentTableView(availableSegmentMetadataMap, publishedSegmentBuilder.build()); + } + + protected DatasourceTable.PhysicalDatasourceMetadata getDatasource(String name) + { + return useSegmentMetadataCache ? segmentMetadataCache.getPhysicalDatasourceMetadata(name) : datasourceSchemaMap.get(name); + } + + protected Set getDatasourceNames() + { + return useSegmentMetadataCache ? segmentMetadataCache.getDatasourceNames() : datasourceSchemaMap.keySet(); + } + + // Note that coordinator must be up to get segments + private JsonParserIterator querySegmentMetadata( + Set watchedDataSources + ) + { + String query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"; + if (watchedDataSources != null && !watchedDataSources.isEmpty()) { + log.debug( + "filtering datasources in published segments based on broker's watchedDataSources[%s]", watchedDataSources); + final StringBuilder sb = new StringBuilder(); + for (String ds : watchedDataSources) { + sb.append("datasources=").append(ds).append("&"); + } + sb.setLength(sb.length() - 1); + query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&" + sb; + } + + return SystemSchema.getThingsFromLeaderNode( + query, + new TypeReference() + { + }, + druidLeaderClient, + objectMapper + ); + } + + private class PollTask implements Runnable + { + @Override + public void run() + { + long delayMS = pollPeriodInMS; + try { + final long pollStartTime = System.nanoTime(); + if (isMetadataSegmentCacheEnabled) { + pollSegmentMetadata(); + } + if (!useSegmentMetadataCache) { + pollDatasourceSchema(); + } + final long pollEndTime = System.nanoTime(); + final long pollTimeNS = pollEndTime - pollStartTime; + final long pollTimeMS = TimeUnit.NANOSECONDS.toMillis(pollTimeNS); + delayMS = Math.max(pollPeriodInMS - pollTimeMS, 0); + } + catch (Exception e) { + log.makeAlert(e, "Problem polling Coordinator.").emit(); + } + finally { + if (!Thread.currentThread().isInterrupted()) { + scheduledExec.schedule(new PollTask(), delayMS, TimeUnit.MILLISECONDS); + } + } + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java index e9c894c91527..8b7d885ed64b 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java @@ -19,7 +19,9 @@ package org.apache.druid.sql.calcite.schema; +import com.google.common.base.Preconditions; import com.google.inject.Binder; +import com.google.inject.Inject; import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.Scopes; @@ -29,6 +31,8 @@ import org.apache.druid.guice.LifecycleModule; import org.apache.druid.sql.guice.SqlBindings; +import java.util.Properties; + /** * The module responsible for providing bindings to Calcite schemas. */ @@ -36,8 +40,16 @@ public class DruidCalciteSchemaModule implements Module { private static final String DRUID_SCHEMA_NAME = "druid"; private static final String INFORMATION_SCHEMA_NAME = "INFORMATION_SCHEMA"; + private static final String SEGMENT_METADATA_CACHE_DISABLED = "druid.sql.planner.disableSegmentMetadataCache"; static final String INCOMPLETE_SCHEMA = "INCOMPLETE_SCHEMA"; + private final Properties properties; + + public DruidCalciteSchemaModule(Properties properties) + { + this.properties = properties; + } + @Override public void configure(Binder binder) { @@ -49,8 +61,10 @@ public void configure(Binder binder) .toProvider(RootSchemaProvider.class) .in(Scopes.SINGLETON); - // SegmentMetadataCache needs to listen to changes for incoming segments - LifecycleModule.register(binder, SegmentMetadataCache.class); + // BrokerSegmentMetadataCache needs to listen to changes for incoming segments + if (!isSegmentMetadataCacheDisabled()) { + LifecycleModule.register(binder, BrokerSegmentMetadataCache.class); + } binder.bind(DruidSchema.class).in(Scopes.SINGLETON); binder.bind(SystemSchema.class).in(Scopes.SINGLETON); binder.bind(InformationSchema.class).in(Scopes.SINGLETON); @@ -70,4 +84,9 @@ private DruidSchemaCatalog getRootSchema(@Named(INCOMPLETE_SCHEMA) DruidSchemaCa rootSchema.getRootSchema().add(INFORMATION_SCHEMA_NAME, informationSchema); return rootSchema; } + + private boolean isSegmentMetadataCacheDisabled() + { + return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_DISABLED)); + } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java index ec0b9cb2c116..28a6ab252516 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java @@ -28,15 +28,16 @@ public class DruidSchema extends AbstractTableSchema { - private final SegmentMetadataCache segmentCache; + private final BrokerSegmentMetadataView brokerSegmentMetadataView; private final DruidSchemaManager druidSchemaManager; @Inject public DruidSchema( - SegmentMetadataCache segmentCache, - final DruidSchemaManager druidSchemaManager) + final BrokerSegmentMetadataView brokerSegmentMetadataView, + final DruidSchemaManager druidSchemaManager + ) { - this.segmentCache = segmentCache; + this.brokerSegmentMetadataView = brokerSegmentMetadataView; if (druidSchemaManager != null && !(druidSchemaManager instanceof NoopDruidSchemaManager)) { this.druidSchemaManager = druidSchemaManager; } else { @@ -44,20 +45,15 @@ public DruidSchema( } } - protected SegmentMetadataCache cache() - { - return segmentCache; - } - @Override public Table getTable(String name) { if (druidSchemaManager != null) { return druidSchemaManager.getTable(name); } else { - DatasourceTable.PhysicalDatasourceMetadata dsMetadata = segmentCache.getDatasource(name); + DatasourceTable.PhysicalDatasourceMetadata dsMetadata = brokerSegmentMetadataView.getDatasource(name); return dsMetadata == null ? null : new DatasourceTable(dsMetadata); - } +} } @Override @@ -66,7 +62,7 @@ public Set getTableNames() if (druidSchemaManager != null) { return druidSchemaManager.getTableNames(); } else { - return segmentCache.getDatasourceNames(); + return brokerSegmentMetadataView.getDatasourceNames(); } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java deleted file mode 100644 index c3a1ab48f5ab..000000000000 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.sql.calcite.schema; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.collect.ImmutableSortedSet; -import com.google.common.util.concurrent.Uninterruptibles; -import com.google.inject.Inject; -import org.apache.druid.client.BrokerSegmentWatcherConfig; -import org.apache.druid.client.DataSegmentInterner; -import org.apache.druid.client.JsonParserIterator; -import org.apache.druid.client.coordinator.Coordinator; -import org.apache.druid.concurrent.LifecycleLock; -import org.apache.druid.discovery.DruidLeaderClient; -import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.lifecycle.LifecycleStop; -import org.apache.druid.java.util.emitter.EmittingLogger; -import org.apache.druid.metadata.SegmentsMetadataManager; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.SegmentStatusInCluster; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - -import java.util.Iterator; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -/** - * This class polls the Coordinator in background to keep the latest published segments. - * Provides {@link #getPublishedSegments()} for others to get segments in metadata store. - * - * The difference between this class and {@link SegmentsMetadataManager} is that this class resides - * in Broker's memory, while {@link SegmentsMetadataManager} resides in Coordinator's memory. In - * fact, this class polls the data from {@link SegmentsMetadataManager} object in the memory of the - * currently leading Coordinator via HTTP queries. - */ -@ManageLifecycle -public class MetadataSegmentView -{ - private static final EmittingLogger log = new EmittingLogger(MetadataSegmentView.class); - - private final DruidLeaderClient coordinatorDruidLeaderClient; - private final ObjectMapper jsonMapper; - private final BrokerSegmentWatcherConfig segmentWatcherConfig; - - private final boolean isCacheEnabled; - /** - * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and - * sys.segments queries return the segments in sorted order based on segmentId. - * - * Volatile since this reference is reassigned in {@code poll()} and then read in {@code getPublishedSegments()} - * from other threads. - */ - @MonotonicNonNull - private volatile ImmutableSortedSet publishedSegments = null; - /** - * Caches the replication factor for segment IDs. In case of coordinator restarts or leadership re-elections, the coordinator API returns `null` replication factor until load rules are evaluated. - * The cache can be used during these periods to continue serving the previously fetched values. - */ - private final Cache segmentIdToReplicationFactor; - private final ScheduledExecutorService scheduledExec; - private final long pollPeriodInMS; - private final LifecycleLock lifecycleLock = new LifecycleLock(); - private final CountDownLatch cachePopulated = new CountDownLatch(1); - - @Inject - public MetadataSegmentView( - final @Coordinator DruidLeaderClient druidLeaderClient, - final ObjectMapper jsonMapper, - final BrokerSegmentWatcherConfig segmentWatcherConfig, - final SegmentMetadataCacheConfig config - ) - { - Preconditions.checkNotNull(config, "SegmentMetadataCacheConfig"); - this.coordinatorDruidLeaderClient = druidLeaderClient; - this.jsonMapper = jsonMapper; - this.segmentWatcherConfig = segmentWatcherConfig; - this.isCacheEnabled = config.isMetadataSegmentCacheEnable(); - this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); - this.scheduledExec = Execs.scheduledSingleThreaded("MetadataSegmentView-Cache--%d"); - this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() - .expireAfterAccess(10, TimeUnit.MINUTES) - .build(); - } - - @LifecycleStart - public void start() - { - if (!lifecycleLock.canStart()) { - throw new ISE("can't start."); - } - try { - if (isCacheEnabled) { - scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); - } - lifecycleLock.started(); - log.info("MetadataSegmentView Started."); - } - finally { - lifecycleLock.exitStart(); - } - } - - @LifecycleStop - public void stop() - { - if (!lifecycleLock.canStop()) { - throw new ISE("can't stop."); - } - log.info("MetadataSegmentView is stopping."); - if (isCacheEnabled) { - scheduledExec.shutdown(); - } - log.info("MetadataSegmentView Stopped."); - } - - private void poll() - { - log.info("polling published segments from coordinator"); - final JsonParserIterator metadataSegments = getMetadataSegments( - coordinatorDruidLeaderClient, - jsonMapper, - segmentWatcherConfig.getWatchedDataSources() - ); - - final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); - while (metadataSegments.hasNext()) { - final SegmentStatusInCluster segment = metadataSegments.next(); - final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); - Integer replicationFactor = segment.getReplicationFactor(); - if (replicationFactor == null) { - replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); - } else { - segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); - } - final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( - interned, - segment.isOvershadowed(), - replicationFactor - ); - builder.add(segmentStatusInCluster); - } - publishedSegments = builder.build(); - cachePopulated.countDown(); - } - - Iterator getPublishedSegments() - { - if (isCacheEnabled) { - Uninterruptibles.awaitUninterruptibly(cachePopulated); - return publishedSegments.iterator(); - } else { - return getMetadataSegments( - coordinatorDruidLeaderClient, - jsonMapper, - segmentWatcherConfig.getWatchedDataSources() - ); - } - } - - // Note that coordinator must be up to get segments - private JsonParserIterator getMetadataSegments( - DruidLeaderClient coordinatorClient, - ObjectMapper jsonMapper, - Set watchedDataSources - ) - { - String query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"; - if (watchedDataSources != null && !watchedDataSources.isEmpty()) { - log.debug( - "filtering datasources in published segments based on broker's watchedDataSources[%s]", watchedDataSources); - final StringBuilder sb = new StringBuilder(); - for (String ds : watchedDataSources) { - sb.append("datasources=").append(ds).append("&"); - } - sb.setLength(sb.length() - 1); - query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&" + sb; - } - - return SystemSchema.getThingsFromLeaderNode( - query, - new TypeReference() - { - }, - coordinatorClient, - jsonMapper - ); - } - - private class PollTask implements Runnable - { - @Override - public void run() - { - long delayMS = pollPeriodInMS; - try { - final long pollStartTime = System.nanoTime(); - poll(); - final long pollEndTime = System.nanoTime(); - final long pollTimeNS = pollEndTime - pollStartTime; - final long pollTimeMS = TimeUnit.NANOSECONDS.toMillis(pollTimeNS); - delayMS = Math.max(pollPeriodInMS - pollTimeMS, 0); - } - catch (Exception e) { - log.makeAlert(e, "Problem polling Coordinator.").emit(); - } - finally { - if (!Thread.currentThread().isInterrupted()) { - scheduledExec.schedule(new PollTask(), delayMS, TimeUnit.MILLISECONDS); - } - } - } - } - -} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java new file mode 100644 index 000000000000..c9760aeb0cf4 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -0,0 +1,42 @@ +package org.apache.druid.sql.calcite.schema; + +import com.google.inject.Inject; +import org.apache.druid.query.GlobalTableDataSource; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.server.SegmentManager; +import org.apache.druid.sql.calcite.table.DatasourceTable; + +public class PhysicalDatasourceMetadataBuilder +{ + private final JoinableFactory joinableFactory; + private final SegmentManager segmentManager; + + @Inject + public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, SegmentManager segmentManager) + { + this.joinableFactory = joinableFactory; + this.segmentManager = segmentManager; + } + + DatasourceTable.PhysicalDatasourceMetadata build(String dataSource, RowSignature rowSignature) + { + final TableDataSource tableDataSource; + + // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing + // in the segment cache, which in this case belongs to the broker meaning only broadcast segments live here) + // to be joinable, it must be possibly joinable according to the factory. we only consider broadcast datasources + // at this time, and isGlobal is currently strongly coupled with joinable, so only make a global table datasource + // if also joinable + final GlobalTableDataSource maybeGlobal = new GlobalTableDataSource(dataSource); + final boolean isJoinable = joinableFactory.isDirectlyJoinable(maybeGlobal); + final boolean isBroadcast = segmentManager.getDataSourceNames().contains(dataSource); + if (isBroadcast && isJoinable) { + tableDataSource = maybeGlobal; + } else { + tableDataSource = new TableDataSource(dataSource); + } + return new DatasourceTable.PhysicalDatasourceMetadata(tableDataSource, rowSignature, isJoinable, isBroadcast); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index db4a10af81c7..f1063c521ef0 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -27,6 +27,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -66,6 +67,7 @@ import org.apache.druid.java.util.http.client.Request; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; @@ -212,7 +214,7 @@ public class SystemSchema extends AbstractSchema @Inject public SystemSchema( final DruidSchema druidSchema, - final MetadataSegmentView metadataView, + final BrokerSegmentMetadataView metadataView, final TimelineServerView serverView, final FilteredServerInventoryView serverInventoryView, final AuthorizerMapper authorizerMapper, @@ -246,11 +248,11 @@ static class SegmentsTable extends AbstractTable implements ScannableTable private final DruidSchema druidSchema; private final ObjectMapper jsonMapper; private final AuthorizerMapper authorizerMapper; - private final MetadataSegmentView metadataView; + private final BrokerSegmentMetadataView metadataView; public SegmentsTable( DruidSchema druidSchemna, - MetadataSegmentView metadataView, + BrokerSegmentMetadataView metadataView, ObjectMapper jsonMapper, AuthorizerMapper authorizerMapper ) @@ -276,15 +278,16 @@ public TableType getJdbcTableType() @Override public Enumerable scan(DataContext root) { - //get available segments from druidSchema + final SegmentTableView segmentTableView = metadataView.getSegmentTableView(); + final Map availableSegmentMetadata = - druidSchema.cache().getSegmentMetadataSnapshot(); + segmentTableView.getAvailableSegmentMetadata(); final Iterator> availableSegmentEntries = availableSegmentMetadata.entrySet().iterator(); // in memory map to store segment data from available segments final Map partialSegmentDataMap = - Maps.newHashMapWithExpectedSize(druidSchema.cache().getTotalSegments()); + Maps.newHashMapWithExpectedSize(segmentTableView.totalSegmentCount()); for (AvailableSegmentMetadata h : availableSegmentMetadata.values()) { PartialSegmentData partialSegmentData = new PartialSegmentData(IS_AVAILABLE_TRUE, h.isRealtime(), h.getNumReplicas(), h.getNumRows()); @@ -293,9 +296,9 @@ public Enumerable scan(DataContext root) // Get published segments from metadata segment cache (if enabled in SQL planner config), else directly from // Coordinator. - final Iterator metadataStoreSegments = metadataView.getPublishedSegments(); + final Iterator metadataStoreSegments = segmentTableView.getPublishedSegments(); - final Set segmentsAlreadySeen = Sets.newHashSetWithExpectedSize(druidSchema.cache().getTotalSegments()); + final Set segmentsAlreadySeen = Sets.newHashSetWithExpectedSize(segmentTableView.totalSegmentCount()); final FluentIterable publishedSegments = FluentIterable .from(() -> getAuthorizedPublishedSegments(metadataStoreSegments, root)) @@ -439,7 +442,40 @@ private Iterator> getAuthorizedAvaila return authorizedSegments.iterator(); } - private static class PartialSegmentData + protected static class SegmentTableView + { + private final Map availableSegmentMetadata; + private final Iterator publishedSegments; + + private final int totalSegmentsCount; + + public SegmentTableView( + Map availableSegmentMetadata, + ImmutableSortedSet publishedSegments + ) + { + this.availableSegmentMetadata = availableSegmentMetadata; + this.publishedSegments = publishedSegments.iterator(); + this.totalSegmentsCount = availableSegmentMetadata.size(); + } + + public Map getAvailableSegmentMetadata() + { + return availableSegmentMetadata; + } + + public Iterator getPublishedSegments() + { + return publishedSegments; + } + + public int totalSegmentCount() + { + return totalSegmentsCount; + } + } + + protected static class PartialSegmentData { private final long isAvailable; private final long isRealtime; diff --git a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java index 56d0d2d5d41f..03a774a05b61 100644 --- a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java +++ b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java @@ -104,7 +104,7 @@ public void configure(Binder binder) binder.bind(TableDefnRegistry.class).in(LazySingleton.class); - binder.install(new DruidCalciteSchemaModule()); + binder.install(new DruidCalciteSchemaModule(props)); binder.install(new CalcitePlannerModule()); binder.install(new SqlAggregationModule()); binder.install(new DruidViewModule()); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java index 858f2e5d8ac9..29de2639ae4a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java @@ -24,7 +24,8 @@ import org.apache.druid.guice.GuiceInjectors; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.JsonConfigurator; -import org.apache.druid.sql.calcite.schema.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.joda.time.Period; import org.junit.Assert; import org.junit.Test; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java index d10bd2e227b8..c1a0d5047ea7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java @@ -21,8 +21,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; @@ -30,7 +31,7 @@ import org.apache.druid.server.SegmentManager; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.NoopEscalator; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; @@ -60,7 +61,7 @@ public void testInitializationWithNoData() throws Exception new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java index 10964800bc43..d34bed3ccf73 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java @@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.BrokerServerView; import org.apache.druid.client.DruidServer; @@ -42,6 +42,9 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.metadata.PhysicalDatasourceMetadata; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; import org.apache.druid.query.TableDataSource; @@ -140,7 +143,7 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -250,7 +253,7 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java index e7eb01a59f95..97534f2f81d9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java @@ -33,7 +33,7 @@ import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.easymock.EasyMock; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java index b54149054895..ae5f14aab8c6 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java @@ -30,7 +30,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; @@ -40,6 +40,8 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryContexts; @@ -70,7 +72,7 @@ import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.NoopEscalator; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.table.DatasourceTable; import org.apache.druid.sql.calcite.table.DruidTable; import org.apache.druid.sql.calcite.util.CalciteTests; @@ -299,7 +301,7 @@ public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheCon ), config, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -337,7 +339,7 @@ public SegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedE ), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -708,7 +710,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -751,7 +753,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -798,7 +800,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -842,7 +844,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -883,7 +885,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -941,7 +943,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -1002,7 +1004,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -1037,7 +1039,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -1085,7 +1087,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ) { @@ -1288,11 +1290,11 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception String brokerInternalQueryConfigJson = "{\"context\": { \"priority\": 5} }"; TestHelper.makeJsonMapper(); - BrokerInternalQueryConfig brokerInternalQueryConfig = MAPPER.readValue( + InternalQueryConfig internalQueryConfig = MAPPER.readValue( MAPPER.writeValueAsString( - MAPPER.readValue(brokerInternalQueryConfigJson, BrokerInternalQueryConfig.class) + MAPPER.readValue(brokerInternalQueryConfigJson, InternalQueryConfig.class) ), - BrokerInternalQueryConfig.class + InternalQueryConfig.class ); DataSegment segment = newSegment("test", 0); @@ -1327,7 +1329,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception ), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - brokerInternalQueryConfig, + internalQueryConfig, new NoopServiceEmitter() ); @@ -1463,7 +1465,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), emitter ) { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index 82eae68c3321..52439c33d6e9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -36,7 +36,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; import org.apache.druid.client.ImmutableDruidDataSource; @@ -64,6 +64,7 @@ import org.apache.druid.java.util.http.client.response.HttpResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; @@ -93,7 +94,7 @@ import org.apache.druid.server.security.AuthorizerMapper; import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.server.security.ResourceType; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable; import org.apache.druid.sql.calcite.table.RowSignatures; import org.apache.druid.sql.calcite.util.CalciteTestBase; @@ -165,7 +166,7 @@ public class SystemSchemaTest extends CalciteTestBase private AuthorizerMapper authMapper; private static QueryRunnerFactoryConglomerate conglomerate; private static Closer resourceCloser; - private MetadataSegmentView metadataView; + private BrokerSegmentMetadataView metadataView; private DruidNodeDiscoveryProvider druidNodeDiscoveryProvider; private FilteredServerInventoryView serverInventoryView; @@ -258,13 +259,13 @@ public void setUp() throws Exception new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ); cache.start(); cache.awaitInitialization(); druidSchema = new DruidSchema(cache, null); - metadataView = EasyMock.createMock(MetadataSegmentView.class); + metadataView = EasyMock.createMock(BrokerSegmentMetadataView.class); druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class); serverInventoryView = EasyMock.createMock(FilteredServerInventoryView.class); schema = new SystemSchema( @@ -567,14 +568,14 @@ public void testSegmentsTable() throws Exception { final SegmentsTable segmentsTable = new SegmentsTable(druidSchema, metadataView, new ObjectMapper(), authMapper); final Set publishedSegments = new HashSet<>(Arrays.asList( - new SegmentStatusInCluster(publishedCompactedSegment1, true, 2), - new SegmentStatusInCluster(publishedCompactedSegment2, false, 0), - new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2), - new SegmentStatusInCluster(segment1, true, 2), - new SegmentStatusInCluster(segment2, false, 0) + new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 2L, 0L, true), + new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 2L, 0L, true), + new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 2L, 0L, true), + new SegmentStatusInCluster(segment1, true, 2, 2L, 0L, true), + new SegmentStatusInCluster(segment2, false, 0, 2L, 0L, true) )); - EasyMock.expect(metadataView.getPublishedSegments()).andReturn(publishedSegments.iterator()).once(); + EasyMock.expect(metadataView.getSegmentMetadata()).andReturn(publishedSegments.iterator()).once(); EasyMock.replay(client, request, responseHolder, responseHandler, metadataView); DataContext dataContext = createDataContext(Users.SUPER); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java index bab3e0957e8f..f182bd5dab58 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java @@ -68,12 +68,12 @@ import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.run.NativeSqlEngine; import org.apache.druid.sql.calcite.run.SqlEngine; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataView; import org.apache.druid.sql.calcite.schema.DruidSchema; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; -import org.apache.druid.sql.calcite.schema.MetadataSegmentView; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.timeline.DataSegment; import org.joda.time.Duration; @@ -372,7 +372,7 @@ public ListenableFuture findCurrentLeader() return new SystemSchema( druidSchema, - new MetadataSegmentView( + new BrokerSegmentMetadataView( druidLeaderClient, getJsonMapper(), new BrokerSegmentWatcherConfig(), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index 63235a76c7e1..1e2b9aab25bf 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -25,7 +25,7 @@ import com.google.inject.Injector; import org.apache.calcite.jdbc.CalciteSchema; import org.apache.calcite.schema.SchemaPlus; -import org.apache.druid.client.BrokerInternalQueryConfig; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.guice.ExpressionModule; import org.apache.druid.java.util.emitter.core.NoopEmitter; import org.apache.druid.java.util.emitter.service.ServiceEmitter; @@ -55,7 +55,7 @@ import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; -import org.apache.druid.sql.calcite.planner.SegmentMetadataCacheConfig; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchema; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; @@ -68,7 +68,7 @@ import org.apache.druid.sql.calcite.schema.NamedSystemSchema; import org.apache.druid.sql.calcite.schema.NamedViewSchema; import org.apache.druid.sql.calcite.schema.NoopDruidSchemaManager; -import org.apache.druid.sql.calcite.schema.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.sql.calcite.schema.ViewSchema; import org.apache.druid.sql.calcite.view.ViewManager; @@ -84,6 +84,7 @@ public class QueryFrameworkUtils { + public static final String INFORMATION_SCHEMA_NAME = "INFORMATION_SCHEMA"; public static QueryLifecycleFactory createMockQueryLifecycleFactory( @@ -224,7 +225,7 @@ public Set getDataSourceNames() createDefaultJoinableFactory(injector), SegmentMetadataCacheConfig.create(), CalciteTests.TEST_AUTHENTICATOR_ESCALATOR, - new BrokerInternalQueryConfig(), + new InternalQueryConfig(), new NoopServiceEmitter() ); @@ -273,5 +274,5 @@ public static DruidOperatorTable createOperatorTable(final Injector injector) public static LookupSchema createMockLookupSchema(final Injector injector) { return new LookupSchema(injector.getInstance(LookupExtractorFactoryContainerProvider.class)); - } + } **/ } From d1c0dca1494f6fdbeff74cd17170232f8c9450ea Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 28 Aug 2023 18:21:45 +0530 Subject: [PATCH 02/36] refactor CoordinatorServerView --- .../druid/client/CoordinatorServerView.java | 478 +----------------- .../client/CoordinatorServerViewImpl.java | 224 ++++++++ .../TimelineAwareCoordinatorServerView.java | 378 ++++++++++++++ ...oordinatorServerViewImplImplImplTest.java} | 8 +- 4 files changed, 612 insertions(+), 476 deletions(-) create mode 100644 server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java create mode 100644 server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java rename server/src/test/java/org/apache/druid/client/{CoordinatorServerViewTest.java => CoordinatorServerViewImplImplImplTest.java} (98%) diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index 45cdc6b11b91..b995ed70b7fa 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -1,481 +1,15 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - package org.apache.druid.client; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Iterables; -import com.google.common.collect.Ordering; -import com.google.inject.Inject; -import org.apache.druid.client.selector.QueryableDruidServer; -import org.apache.druid.client.selector.ServerSelector; -import org.apache.druid.client.selector.TierSelectorStrategy; -import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.guice.annotations.EscalatedClient; -import org.apache.druid.guice.annotations.Smile; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; -import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.client.InventoryView; +import org.apache.druid.client.SegmentLoadInfo; import org.apache.druid.query.DataSource; -import org.apache.druid.query.QueryRunner; -import org.apache.druid.query.QueryToolChestWarehouse; -import org.apache.druid.query.QueryWatcher; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.planning.DataSourceAnalysis; -import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.server.coordination.ServerType; -import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.TimelineLookup; import org.apache.druid.timeline.VersionedIntervalTimeline; -import org.apache.druid.timeline.partition.PartitionChunk; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.function.Function; -import java.util.stream.Collectors; -/** - * ServerView of coordinator for the state of segments being loaded in the cluster. - */ -@ManageLifecycle -public class CoordinatorServerView implements InventoryView, TimelineServerView +public interface CoordinatorServerView extends InventoryView { - private static final Logger log = new Logger(CoordinatorServerView.class); - - private final Object lock = new Object(); - - private final Map segmentLoadInfos; - private final ConcurrentMap clients = new ConcurrentHashMap<>(); - private final Map selectors = new HashMap<>(); - private final Map> timelines2 = new HashMap<>(); - private final Map> timelines; - - private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); - - private final ServerInventoryView baseView; - private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; - - private final CountDownLatch initialized = new CountDownLatch(1); - private final ServiceEmitter emitter; - - private final TierSelectorStrategy tierSelectorStrategy; - - private final QueryToolChestWarehouse warehouse; - private final QueryWatcher queryWatcher; - private final ObjectMapper smileMapper; - private final HttpClient httpClient; - - @Inject - public CoordinatorServerView( - final QueryToolChestWarehouse warehouse, - final QueryWatcher queryWatcher, - final @Smile ObjectMapper smileMapper, - final @EscalatedClient HttpClient httpClient, - ServerInventoryView baseView, - CoordinatorSegmentWatcherConfig segmentWatcherConfig, - ServiceEmitter emitter, - TierSelectorStrategy tierSelectorStrategy - ) - { - this.baseView = baseView; - this.segmentWatcherConfig = segmentWatcherConfig; - this.emitter = emitter; - this.segmentLoadInfos = new HashMap<>(); - this.timelines = new HashMap<>(); - this.tierSelectorStrategy = tierSelectorStrategy; - this.warehouse = warehouse; - this.queryWatcher = queryWatcher; - this.smileMapper = smileMapper; - this.httpClient = httpClient; - - ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); - baseView.registerSegmentCallback( - exec, - new ServerView.SegmentCallback() - { - @Override - public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) - { - serverAddedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) - { - serverRemovedSegment2(server, segment); - serverRemovedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentViewInitialized() - { - initialized.countDown(); - runTimelineCallbacks(TimelineCallback::timelineInitialized); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - - baseView.registerServerRemovedCallback( - exec, - new ServerView.ServerRemovedCallback() - { - @Override - public ServerView.CallbackAction serverRemoved(DruidServer server) - { - removeServer(server); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - } - - @LifecycleStart - public void start() throws InterruptedException - { - if (segmentWatcherConfig.isAwaitInitializationOnStart()) { - final long startMillis = System.currentTimeMillis(); - log.info("%s waiting for initialization.", getClass().getSimpleName()); - initialized.await(); - final long endMillis = System.currentTimeMillis(); - log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().build( - "serverview/init/time", - endMillis - startMillis - )); - } - } - - private void removeServer(DruidServer server) - { - for (DataSegment segment : server.iterateAllSegments()) { - serverAddedSegment2(server.getMetadata(), segment); - serverRemovedSegment(server.getMetadata(), segment); - } - } - - private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) - { - SegmentId segmentId = segment.getId(); - synchronized (lock) { - log.debug("Adding segment[%s] for server[%s]", segment, server); - - SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - // servers escape the scope of this object so use ConcurrentSet - segmentLoadInfo = new SegmentLoadInfo(segment); - - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - if (timeline == null) { - timeline = new VersionedIntervalTimeline<>(Ordering.natural()); - timelines.put(segment.getDataSource(), timeline); - } - - timeline.add( - segment.getInterval(), - segment.getVersion(), - segment.getShardSpec().createChunk(segmentLoadInfo) - ); - segmentLoadInfos.put(segmentId, segmentLoadInfo); - } - segmentLoadInfo.addServer(server); - } - runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); - } - - private void serverAddedSegment2(final DruidServerMetadata server, final DataSegment segment) - { - final SegmentId segmentId = segment.getId(); - synchronized (lock) { - // in theory we could probably just filter this to ensure we don't put ourselves in here, to make broker tree - // query topologies, but for now just skip all brokers, so we don't create some sort of wild infinite query - // loop... - if (!server.getType().equals(ServerType.BROKER)) { - log.debug("Adding segment[%s] for server[%s]", segment, server); - ServerSelector selector = selectors.get(segmentId); - if (selector == null) { - selector = new ServerSelector(segment, tierSelectorStrategy); - - VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); - if (timeline == null) { - // broker needs to skip tombstones - timeline = new VersionedIntervalTimeline<>(Ordering.natural(), true); - timelines2.put(segment.getDataSource(), timeline); - } - - timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); - selectors.put(segmentId, selector); - } - - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); - if (inventoryValue == null) { - log.warn( - "Could not find server[%s] in inventory. Skipping addition of segment[%s].", - server.getName(), - segmentId - ); - return; - } else { - queryableDruidServer = addServer(inventoryValue); - } - } - selector.addServerAndUpdateSegment(queryableDruidServer, segment); - } - // run the callbacks, even if the segment came from a broker, lets downstream watchers decide what to do with it - runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); - } - } - - private QueryableDruidServer addServer(DruidServer server) - { - QueryableDruidServer retVal = new QueryableDruidServer<>(server, makeDirectClient(server)); - QueryableDruidServer exists = clients.put(server.getName(), retVal); - if (exists != null) { - log.warn("QueryRunner for server[%s] already exists!? Well it's getting replaced", server); - } - - return retVal; - } - - private DirectDruidClient makeDirectClient(DruidServer server) - { - return new DirectDruidClient( - warehouse, - queryWatcher, - smileMapper, - httpClient, - server.getScheme(), - server.getHost(), - emitter - ); - } - - private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) - { - SegmentId segmentId = segment.getId(); - - synchronized (lock) { - log.debug("Removing segment[%s] from server[%s].", segmentId, server); - - final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - log.warn("Told to remove non-existant segment[%s]", segmentId); - return; - } - if (segmentLoadInfo.removeServer(server)) { - runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); - } - if (segmentLoadInfo.isEmpty()) { - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - segmentLoadInfos.remove(segmentId); - - final PartitionChunk removedPartition = timeline.remove( - segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( - new SegmentLoadInfo( - segment - ) - ) - ); - - if (removedPartition == null) { - log.warn( - "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", - segment.getInterval(), - segment.getVersion() - ); - } else { - runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); - } - } - } - } - - private void serverRemovedSegment2(DruidServerMetadata server, DataSegment segment) - { - final SegmentId segmentId = segment.getId(); - final ServerSelector selector; - - synchronized (lock) { - log.debug("Removing segment[%s] from server[%s].", segmentId, server); - - // we don't store broker segments here, but still run the callbacks for the segment being removed from the server - // since the broker segments are not stored on the timeline, do not fire segmentRemoved event - selector = selectors.get(segmentId); - if (selector == null) { - log.warn("Told to remove non-existant segment[%s]", segmentId); - return; - } - - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - log.warn( - "Could not find server[%s] in inventory. Skipping removal of segment[%s].", - server.getName(), - segmentId - ); - } else if (!selector.removeServer(queryableDruidServer)) { - log.warn( - "Asked to disassociate non-existant association between server[%s] and segment[%s]", - server, - segmentId - ); - } else { - // runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); - } - - if (selector.isEmpty()) { - VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); - selectors.remove(segmentId); - - final PartitionChunk removedPartition = timeline.remove( - segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector) - ); - - if (removedPartition == null) { - log.warn( - "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", - segment.getInterval(), - segment.getVersion() - ); - } else { - // runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); - } - } - } - } - - public VersionedIntervalTimeline getTimeline(DataSource dataSource) - { - String table = Iterables.getOnlyElement(dataSource.getTableNames()); - synchronized (lock) { - return timelines.get(table); - } - } - - public Set getLoadedSegmentIds() - { - return segmentLoadInfos.keySet(); - } - - @Override - public DruidServer getInventoryValue(String serverKey) - { - return baseView.getInventoryValue(serverKey); - } - - @Override - public Collection getInventory() - { - return baseView.getInventory(); - } - - @Override - public boolean isStarted() - { - return baseView.isStarted(); - } - - @Override - public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) - { - return baseView.isSegmentLoadedByServer(serverKey, segment); - } - - private void runTimelineCallbacks(final Function function) - { - for (Map.Entry entry : timelineCallbacks.entrySet()) { - entry.getValue().execute( - () -> { - if (CallbackAction.UNREGISTER == function.apply(entry.getKey())) { - timelineCallbacks.remove(entry.getKey()); - } - } - ); - } - } - - @Override - public void registerTimelineCallback(Executor exec, TimelineCallback callback) - { - timelineCallbacks.put(callback, exec); - } - - @Override - public void registerServerRemovedCallback(Executor exec, ServerRemovedCallback callback) - { - throw new UnsupportedOperationException(); - } - - @Override - public void registerSegmentCallback(Executor exec, SegmentCallback callback) - { - baseView.registerSegmentCallback(exec, callback); - } - - @Override - public Optional> getTimeline(DataSourceAnalysis analysis) - { - final TableDataSource table = - analysis.getBaseTableDataSource() - .orElseThrow(() -> new ISE("Cannot handle base datasource: %s", analysis.getBaseDataSource())); - - synchronized (lock) { - return Optional.ofNullable(timelines2.get(table.getName())); - } - } - - @Override - public List getDruidServers() - { - return clients.values().stream() - .map(queryableDruidServer -> queryableDruidServer.getServer().toImmutableDruidServer()) - .collect(Collectors.toList()); - } - - @Override - public QueryRunner getQueryRunner(DruidServer server) - { - synchronized (lock) { - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - log.error("No QueryRunner found for server name[%s].", server.getName()); - return null; - } - return queryableDruidServer.getQueryRunner(); - } - } -} + VersionedIntervalTimeline getTimeline(DataSource dataSource); + Map getSegmentLoadInfos(); +} \ No newline at end of file diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java new file mode 100644 index 000000000000..cfcab66105bd --- /dev/null +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java @@ -0,0 +1,224 @@ +package org.apache.druid.client; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.inject.Inject; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import org.apache.druid.query.DataSource; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.PartitionChunk; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; + +/** + * ServerView of coordinator for the state of segments being loaded in the cluster. + */ +@ManageLifecycle +public class CoordinatorServerViewImpl implements CoordinatorServerView +{ + private static final Logger log = new Logger(Alpha.class); + + private final Object lock = new Object(); + + private final Map segmentLoadInfos; + private final Map> timelines; + + private final ServerInventoryView baseView; + private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; + + private final CountDownLatch initialized = new CountDownLatch(1); + private final ServiceEmitter emitter; + + @Inject + public CoordinatorServerViewImpl( + ServerInventoryView baseView, + CoordinatorSegmentWatcherConfig segmentWatcherConfig, + ServiceEmitter emitter + ) + { + this.baseView = baseView; + this.segmentWatcherConfig = segmentWatcherConfig; + this.emitter = emitter; + this.segmentLoadInfos = new HashMap<>(); + this.timelines = new HashMap<>(); + + ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); + baseView.registerSegmentCallback( + exec, + new ServerView.SegmentCallback() + { + @Override + public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) + { + serverAddedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) + { + serverRemovedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentViewInitialized() + { + initialized.countDown(); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + + baseView.registerServerRemovedCallback( + exec, + new ServerView.ServerRemovedCallback() + { + @Override + public ServerView.CallbackAction serverRemoved(DruidServer server) + { + removeServer(server); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + } + + @LifecycleStart + public void start() throws InterruptedException + { + if (segmentWatcherConfig.isAwaitInitializationOnStart()) { + final long startMillis = System.currentTimeMillis(); + log.info("%s waiting for initialization.", getClass().getSimpleName()); + initialized.await(); + final long endMillis = System.currentTimeMillis(); + log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); + emitter.emit(ServiceMetricEvent.builder().build( + "serverview/init/time", + endMillis - startMillis + )); + } + } + + private void removeServer(DruidServer server) + { + for (DataSegment segment : server.iterateAllSegments()) { + serverRemovedSegment(server.getMetadata(), segment); + } + } + + private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) + { + SegmentId segmentId = segment.getId(); + synchronized (lock) { + log.debug("Adding segment[%s] for server[%s]", segment, server); + + SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + // servers escape the scope of this object so use ConcurrentSet + segmentLoadInfo = new SegmentLoadInfo(segment); + + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + if (timeline == null) { + timeline = new VersionedIntervalTimeline<>(Ordering.natural()); + timelines.put(segment.getDataSource(), timeline); + } + + timeline.add( + segment.getInterval(), + segment.getVersion(), + segment.getShardSpec().createChunk(segmentLoadInfo) + ); + segmentLoadInfos.put(segmentId, segmentLoadInfo); + } + segmentLoadInfo.addServer(server); + } + } + + private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) + { + SegmentId segmentId = segment.getId(); + + synchronized (lock) { + log.debug("Removing segment[%s] from server[%s].", segmentId, server); + + final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + log.warn("Told to remove non-existant segment[%s]", segmentId); + return; + } + segmentLoadInfo.removeServer(server); + if (segmentLoadInfo.isEmpty()) { + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + segmentLoadInfos.remove(segmentId); + + final PartitionChunk removedPartition = timeline.remove( + segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( + new SegmentLoadInfo( + segment + ) + ) + ); + + if (removedPartition == null) { + log.warn( + "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", + segment.getInterval(), + segment.getVersion() + ); + } + } + } + } + + @Override + public VersionedIntervalTimeline getTimeline(DataSource dataSource) + { + String table = Iterables.getOnlyElement(dataSource.getTableNames()); + synchronized (lock) { + return timelines.get(table); + } + } + + @Override + public Map getSegmentLoadInfos() + { + return segmentLoadInfos; + } + + @Override + public DruidServer getInventoryValue(String serverKey) + { + return baseView.getInventoryValue(serverKey); + } + + @Override + public Collection getInventory() + { + return baseView.getInventory(); + } + + @Override + public boolean isStarted() + { + return baseView.isStarted(); + } + + @Override + public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) + { + return baseView.isSegmentLoadedByServer(serverKey, segment); + } +} diff --git a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java new file mode 100644 index 000000000000..a58c2e18957f --- /dev/null +++ b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java @@ -0,0 +1,378 @@ +package org.apache.druid.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.inject.Inject; +import org.apache.druid.client.selector.QueryableDruidServer; +import org.apache.druid.client.selector.ServerSelector; +import org.apache.druid.client.selector.TierSelectorStrategy; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.guice.annotations.EscalatedClient; +import org.apache.druid.guice.annotations.Smile; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.QueryWatcher; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.planning.DataSourceAnalysis; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.TimelineLookup; +import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.PartitionChunk; +import org.apache.druid.utils.CollectionUtils; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.function.Function; +import java.util.stream.Collectors; + +@ManageLifecycle +public class TimelineAwareCoordinatorServerView implements CoordinatorServerView, TimelineServerView +{ + private static final Logger log = new Logger(TimelineAwareCoordinatorServerView.class); + + private final Object lock = new Object(); + + private final ConcurrentMap clients = new ConcurrentHashMap<>(); + private final Map selectors = new HashMap<>(); + private final Map> timelines = new HashMap<>(); + + private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); + + private final ServerInventoryView baseView; + private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; + + private final CountDownLatch initialized = new CountDownLatch(1); + private final ServiceEmitter emitter; + + private final TierSelectorStrategy tierSelectorStrategy; + + private final QueryToolChestWarehouse warehouse; + private final QueryWatcher queryWatcher; + private final ObjectMapper smileMapper; + private final HttpClient httpClient; + + @Inject + public TimelineAwareCoordinatorServerView( + final QueryToolChestWarehouse warehouse, + final QueryWatcher queryWatcher, + final @Smile ObjectMapper smileMapper, + final @EscalatedClient HttpClient httpClient, + ServerInventoryView baseView, + CoordinatorSegmentWatcherConfig segmentWatcherConfig, + ServiceEmitter emitter, + TierSelectorStrategy tierSelectorStrategy + ) + { + this.baseView = baseView; + this.segmentWatcherConfig = segmentWatcherConfig; + this.emitter = emitter; + this.tierSelectorStrategy = tierSelectorStrategy; + this.warehouse = warehouse; + this.queryWatcher = queryWatcher; + this.smileMapper = smileMapper; + this.httpClient = httpClient; + + ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); + baseView.registerSegmentCallback( + exec, + new ServerView.SegmentCallback() + { + @Override + public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) + { + serverAddedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) + { + serverRemovedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentViewInitialized() + { + initialized.countDown(); + runTimelineCallbacks(TimelineCallback::timelineInitialized); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + + baseView.registerServerRemovedCallback( + exec, + new ServerView.ServerRemovedCallback() + { + @Override + public ServerView.CallbackAction serverRemoved(DruidServer server) + { + removeServer(server); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + } + + @LifecycleStart + public void start() throws InterruptedException + { + if (segmentWatcherConfig.isAwaitInitializationOnStart()) { + final long startMillis = System.currentTimeMillis(); + log.info("%s waiting for initialization.", getClass().getSimpleName()); + initialized.await(); + final long endMillis = System.currentTimeMillis(); + log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); + emitter.emit(ServiceMetricEvent.builder().build( + "serverview/init/time", + endMillis - startMillis + )); + } + } + + private void removeServer(DruidServer server) + { + for (DataSegment segment : server.iterateAllSegments()) { + serverRemovedSegment(server.getMetadata(), segment); + } + } + + private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) + { + final SegmentId segmentId = segment.getId(); + synchronized (lock) { + log.debug("Adding segment[%s] for server[%s]", segment, server); + ServerSelector selector = selectors.get(segmentId); + if (selector == null) { + selector = new ServerSelector(segment, tierSelectorStrategy); + + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + if (timeline == null) { + // broker needs to skip tombstones + timeline = new VersionedIntervalTimeline<>(Ordering.natural()); + timelines.put(segment.getDataSource(), timeline); + } + + timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); + selectors.put(segmentId, selector); + } + + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); + if (inventoryValue == null) { + log.warn( + "Could not find server[%s] in inventory. Skipping addition of segment[%s].", + server.getName(), + segmentId + ); + return; + } else { + queryableDruidServer = addServer(inventoryValue); + } + } + selector.addServerAndUpdateSegment(queryableDruidServer, segment); + // run the callbacks, even if the segment came from a broker, lets downstream watchers decide what to do with it + runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); + } + } + + private QueryableDruidServer addServer(DruidServer server) + { + QueryableDruidServer retVal = new QueryableDruidServer<>(server, makeDirectClient(server)); + QueryableDruidServer exists = clients.put(server.getName(), retVal); + if (exists != null) { + log.warn("QueryRunner for server[%s] already exists!? Well it's getting replaced", server); + } + + return retVal; + } + + private DirectDruidClient makeDirectClient(DruidServer server) + { + return new DirectDruidClient( + warehouse, + queryWatcher, + smileMapper, + httpClient, + server.getScheme(), + server.getHost(), + emitter + ); + } + + private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) + { + final SegmentId segmentId = segment.getId(); + final ServerSelector selector; + + synchronized (lock) { + log.debug("Removing segment[%s] from server[%s].", segmentId, server); + + selector = selectors.get(segmentId); + if (selector == null) { + log.warn("Told to remove non-existant segment[%s]", segmentId); + return; + } + + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + log.warn( + "Could not find server[%s] in inventory. Skipping removal of segment[%s].", + server.getName(), + segmentId + ); + } else if (!selector.removeServer(queryableDruidServer)) { + log.warn( + "Asked to disassociate non-existant association between server[%s] and segment[%s]", + server, + segmentId + ); + } else { + runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); + } + + if (selector.isEmpty()) { + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + selectors.remove(segmentId); + + final PartitionChunk removedPartition = timeline.remove( + segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector) + ); + + if (removedPartition == null) { + log.warn( + "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", + segment.getInterval(), + segment.getVersion() + ); + } else { + runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); + } + } + } + } + + @Override + public VersionedIntervalTimeline getTimeline(DataSource dataSource) + { + String table = Iterables.getOnlyElement(dataSource.getTableNames()); + synchronized (lock) { + timelines.get(table).getAllTimelineEntries() + } + } + + @Override + public Map getSegmentLoadInfos() + { + return CollectionUtils.mapValues(selectors, ServerSelector::toSegmentLoadInfo); + } + + @Override + public DruidServer getInventoryValue(String serverKey) + { + return baseView.getInventoryValue(serverKey); + } + + @Override + public Collection getInventory() + { + return baseView.getInventory(); + } + + @Override + public boolean isStarted() + { + return baseView.isStarted(); + } + + @Override + public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) + { + return baseView.isSegmentLoadedByServer(serverKey, segment); + } + + private void runTimelineCallbacks(final Function function) + { + for (Map.Entry entry : timelineCallbacks.entrySet()) { + entry.getValue().execute( + () -> { + if (CallbackAction.UNREGISTER == function.apply(entry.getKey())) { + timelineCallbacks.remove(entry.getKey()); + } + } + ); + } + } + + @Override + public void registerTimelineCallback(Executor exec, TimelineCallback callback) + { + timelineCallbacks.put(callback, exec); + } + + @Override + public void registerServerRemovedCallback(Executor exec, ServerRemovedCallback callback) + { + throw new UnsupportedOperationException(); + } + + @Override + public void registerSegmentCallback(Executor exec, SegmentCallback callback) + { + baseView.registerSegmentCallback(exec, callback); + } + + @Override + public Optional> getTimeline(DataSourceAnalysis analysis) + { + final TableDataSource table = + analysis.getBaseTableDataSource() + .orElseThrow(() -> new ISE("Cannot handle base datasource: %s", analysis.getBaseDataSource())); + + synchronized (lock) { + return Optional.ofNullable(timelines.get(table.getName())); + } + } + + @Override + public List getDruidServers() + { + return clients.values().stream() + .map(queryableDruidServer -> queryableDruidServer.getServer().toImmutableDruidServer()) + .collect(Collectors.toList()); + } + + @Override + public QueryRunner getQueryRunner(DruidServer server) + { + synchronized (lock) { + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + log.error("No QueryRunner found for server name[%s].", server.getName()); + return null; + } + return queryableDruidServer.getQueryRunner(); + } + } +} diff --git a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java similarity index 98% rename from server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java rename to server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java index 1bca2a0690af..1614709fce7d 100644 --- a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java +++ b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java @@ -52,7 +52,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; -public class CoordinatorServerViewTest extends CuratorTestBase +public class CoordinatorServerViewImplImplTest extends CuratorTestBase { private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; @@ -63,9 +63,9 @@ public class CoordinatorServerViewTest extends CuratorTestBase private CountDownLatch segmentRemovedLatch; private BatchServerInventoryView baseView; - private CoordinatorServerView overlordServerView; + private CoordinatorServerViewImpl overlordServerView; - public CoordinatorServerViewTest() + public CoordinatorServerViewImplImplTest() { jsonMapper = TestHelper.makeJsonMapper(); zkPathsConfig = new ZkPathsConfig(); @@ -332,7 +332,7 @@ public CallbackAction segmentViewInitialized() } }; - overlordServerView = new CoordinatorServerView( + overlordServerView = new CoordinatorServerViewImpl( baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter() From a48d2fe271fff95d9f4650592548286d8f62ffa8 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 28 Aug 2023 18:30:24 +0530 Subject: [PATCH 03/36] minor change --- .../java/org/apache/druid/client/CoordinatorServerViewImpl.java | 2 +- ...ViewImplImplImplTest.java => CoordinatorServerViewTest.java} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename server/src/test/java/org/apache/druid/client/{CoordinatorServerViewImplImplImplTest.java => CoordinatorServerViewTest.java} (99%) diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java index cfcab66105bd..deab4e32e897 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java @@ -28,7 +28,7 @@ @ManageLifecycle public class CoordinatorServerViewImpl implements CoordinatorServerView { - private static final Logger log = new Logger(Alpha.class); + private static final Logger log = new Logger(CoordinatorServerViewImpl.class); private final Object lock = new Object(); diff --git a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java similarity index 99% rename from server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java rename to server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java index 1614709fce7d..11a38af228b6 100644 --- a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewImplImplImplTest.java +++ b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java @@ -52,7 +52,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; -public class CoordinatorServerViewImplImplTest extends CuratorTestBase +public class CoordinatorServerViewTest extends CuratorTestBase { private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; From 5e7756f784b8aaeb2d514acb88481aeb7926f7c2 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 28 Aug 2023 18:37:26 +0530 Subject: [PATCH 04/36] Revert "refactor CoordinatorServerView" This reverts commit d1c0dca1494f6fdbeff74cd17170232f8c9450ea. --- .../timeline/SegmentStatusInCluster.java | 16 +- .../druid/client/CoordinatorServerView.java | 221 ++++++++++++++++- .../client/CoordinatorServerViewImpl.java | 224 ------------------ .../CoordinatorServerViewInterface.java | 14 ++ .../TimelineAwareCoordinatorServerView.java | 170 ++++++++++--- .../druid/client/selector/ServerSelector.java | 8 + .../metadata/AvailableSegmentMetadata.java | 4 +- .../segment/metadata/DatasourceSchema.java | 1 + .../metadata/SegmentMetadataCache.java | 15 +- .../server/http/DataSourcesResource.java | 20 +- .../druid/server/http/MetadataResource.java | 51 ++-- ...melineAwareCoordinatorServerViewTest.java} | 8 +- .../CuratorDruidCoordinatorTest.java | 6 +- .../server/http/DataSourcesResourceTest.java | 6 +- .../server/http/ServersResourceTest.java | 4 +- .../org/apache/druid/cli/CliCoordinator.java | 9 +- .../schema/BrokerSegmentMetadataCache.java | 21 ++ .../schema/BrokerSegmentMetadataView.java | 14 +- .../sql/calcite/schema/SystemSchema.java | 3 +- 19 files changed, 492 insertions(+), 323 deletions(-) delete mode 100644 server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java create mode 100644 server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java rename server/src/test/java/org/apache/druid/client/{CoordinatorServerViewTest.java => TimelineAwareCoordinatorServerViewTest.java} (98%) diff --git a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java index 26d89d79ed34..263f1619c6f3 100644 --- a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java +++ b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java @@ -67,12 +67,12 @@ public SegmentStatusInCluster( @JsonProperty("overshadowed") boolean overshadowed, @JsonProperty("replicationFactor") @Nullable Integer replicationFactor, @JsonProperty("numRows") @Nullable Long numRows, - @JsonProperty("isRealtime") Long isRealtime, - @JsonProperty("isPublished") boolean isPublished + @JsonProperty("isPublished") boolean isPublished, + @JsonProperty("isRealtime") Long isRealtime ) { // Jackson will overwrite dataSegment if needed (even though the field is final) - this(null, overshadowed, replicationFactor, numRows, isRealtime, isPublished); + this(null, overshadowed, replicationFactor, numRows, isPublished, isRealtime); } public SegmentStatusInCluster( @@ -80,8 +80,8 @@ public SegmentStatusInCluster( boolean overshadowed, Integer replicationFactor, Long numRows, - Long isRealtime, - boolean isPublished + boolean isPublished, + Long isRealtime ) { this.dataSegment = dataSegment; @@ -144,13 +144,14 @@ public boolean equals(Object o) && Objects.equals(replicationFactor, that.replicationFactor) && Objects.equals(dataSegment, that.dataSegment) && Objects.equals(numRows, that.numRows) - && Objects.equals(isRealtime, that.isRealtime); + && Objects.equals(isRealtime, that.isRealtime) + && isPublished == that.isPublished; } @Override public int hashCode() { - return Objects.hash(overshadowed, replicationFactor, dataSegment, numRows, isRealtime); + return Objects.hash(overshadowed, replicationFactor, dataSegment, numRows, isPublished, isRealtime); } @Override @@ -167,6 +168,7 @@ public String toString() ", replicationFactor=" + replicationFactor + ", dataSegment=" + dataSegment + ", numRows=" + numRows + + ", isPublished=" + isPublished + ", isRealtime=" + isRealtime + '}'; } diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index b995ed70b7fa..6b6fecc2af6b 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -1,15 +1,224 @@ package org.apache.druid.client; -import org.apache.druid.client.InventoryView; -import org.apache.druid.client.SegmentLoadInfo; +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.inject.Inject; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.query.DataSource; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.VersionedIntervalTimeline; +import org.apache.druid.timeline.partition.PartitionChunk; +import java.util.Collection; +import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; -public interface CoordinatorServerView extends InventoryView +/** + * ServerView of coordinator for the state of segments being loaded in the cluster. + */ +@ManageLifecycle +public class CoordinatorServerView implements CoordinatorServerViewInterface { - VersionedIntervalTimeline getTimeline(DataSource dataSource); - Map getSegmentLoadInfos(); -} \ No newline at end of file + private static final Logger log = new Logger(CoordinatorServerView.class); + + private final Object lock = new Object(); + + private final Map segmentLoadInfos; + private final Map> timelines; + + private final ServerInventoryView baseView; + private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; + + private final CountDownLatch initialized = new CountDownLatch(1); + private final ServiceEmitter emitter; + + @Inject + public CoordinatorServerView( + ServerInventoryView baseView, + CoordinatorSegmentWatcherConfig segmentWatcherConfig, + ServiceEmitter emitter + ) + { + this.baseView = baseView; + this.segmentWatcherConfig = segmentWatcherConfig; + this.emitter = emitter; + this.segmentLoadInfos = new HashMap<>(); + this.timelines = new HashMap<>(); + + ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); + baseView.registerSegmentCallback( + exec, + new ServerView.SegmentCallback() + { + @Override + public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) + { + serverAddedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) + { + serverRemovedSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentViewInitialized() + { + initialized.countDown(); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + + baseView.registerServerRemovedCallback( + exec, + new ServerView.ServerRemovedCallback() + { + @Override + public ServerView.CallbackAction serverRemoved(DruidServer server) + { + removeServer(server); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + } + + @LifecycleStart + public void start() throws InterruptedException + { + if (segmentWatcherConfig.isAwaitInitializationOnStart()) { + final long startMillis = System.currentTimeMillis(); + log.info("%s waiting for initialization.", getClass().getSimpleName()); + initialized.await(); + final long endMillis = System.currentTimeMillis(); + log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); + emitter.emit(ServiceMetricEvent.builder().build( + "serverview/init/time", + endMillis - startMillis + )); + } + } + + private void removeServer(DruidServer server) + { + for (DataSegment segment : server.iterateAllSegments()) { + serverRemovedSegment(server.getMetadata(), segment); + } + } + + private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) + { + SegmentId segmentId = segment.getId(); + synchronized (lock) { + log.debug("Adding segment[%s] for server[%s]", segment, server); + + SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + // servers escape the scope of this object so use ConcurrentSet + segmentLoadInfo = new SegmentLoadInfo(segment); + + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + if (timeline == null) { + timeline = new VersionedIntervalTimeline<>(Ordering.natural()); + timelines.put(segment.getDataSource(), timeline); + } + + timeline.add( + segment.getInterval(), + segment.getVersion(), + segment.getShardSpec().createChunk(segmentLoadInfo) + ); + segmentLoadInfos.put(segmentId, segmentLoadInfo); + } + segmentLoadInfo.addServer(server); + } + } + + private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) + { + SegmentId segmentId = segment.getId(); + + synchronized (lock) { + log.debug("Removing segment[%s] from server[%s].", segmentId, server); + + final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + log.warn("Told to remove non-existant segment[%s]", segmentId); + return; + } + segmentLoadInfo.removeServer(server); + if (segmentLoadInfo.isEmpty()) { + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + segmentLoadInfos.remove(segmentId); + + final PartitionChunk removedPartition = timeline.remove( + segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( + new SegmentLoadInfo( + segment + ) + ) + ); + + if (removedPartition == null) { + log.warn( + "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", + segment.getInterval(), + segment.getVersion() + ); + } + } + } + } + + @Override + public VersionedIntervalTimeline getTimeline(DataSource dataSource) + { + String table = Iterables.getOnlyElement(dataSource.getTableNames()); + synchronized (lock) { + return timelines.get(table); + } + } + + @Override + public Map getSegmentLoadInfos() + { + return segmentLoadInfos; + } + + @Override + public DruidServer getInventoryValue(String serverKey) + { + return baseView.getInventoryValue(serverKey); + } + + @Override + public Collection getInventory() + { + return baseView.getInventory(); + } + + @Override + public boolean isStarted() + { + return baseView.isStarted(); + } + + @Override + public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) + { + return baseView.isSegmentLoadedByServer(serverKey, segment); + } +} diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java deleted file mode 100644 index deab4e32e897..000000000000 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewImpl.java +++ /dev/null @@ -1,224 +0,0 @@ -package org.apache.druid.client; - -import com.google.common.collect.Iterables; -import com.google.common.collect.Ordering; -import com.google.inject.Inject; -import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; -import org.apache.druid.query.DataSource; -import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.VersionedIntervalTimeline; -import org.apache.druid.timeline.partition.PartitionChunk; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; - -/** - * ServerView of coordinator for the state of segments being loaded in the cluster. - */ -@ManageLifecycle -public class CoordinatorServerViewImpl implements CoordinatorServerView -{ - private static final Logger log = new Logger(CoordinatorServerViewImpl.class); - - private final Object lock = new Object(); - - private final Map segmentLoadInfos; - private final Map> timelines; - - private final ServerInventoryView baseView; - private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; - - private final CountDownLatch initialized = new CountDownLatch(1); - private final ServiceEmitter emitter; - - @Inject - public CoordinatorServerViewImpl( - ServerInventoryView baseView, - CoordinatorSegmentWatcherConfig segmentWatcherConfig, - ServiceEmitter emitter - ) - { - this.baseView = baseView; - this.segmentWatcherConfig = segmentWatcherConfig; - this.emitter = emitter; - this.segmentLoadInfos = new HashMap<>(); - this.timelines = new HashMap<>(); - - ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); - baseView.registerSegmentCallback( - exec, - new ServerView.SegmentCallback() - { - @Override - public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) - { - serverAddedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) - { - serverRemovedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentViewInitialized() - { - initialized.countDown(); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - - baseView.registerServerRemovedCallback( - exec, - new ServerView.ServerRemovedCallback() - { - @Override - public ServerView.CallbackAction serverRemoved(DruidServer server) - { - removeServer(server); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - } - - @LifecycleStart - public void start() throws InterruptedException - { - if (segmentWatcherConfig.isAwaitInitializationOnStart()) { - final long startMillis = System.currentTimeMillis(); - log.info("%s waiting for initialization.", getClass().getSimpleName()); - initialized.await(); - final long endMillis = System.currentTimeMillis(); - log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().build( - "serverview/init/time", - endMillis - startMillis - )); - } - } - - private void removeServer(DruidServer server) - { - for (DataSegment segment : server.iterateAllSegments()) { - serverRemovedSegment(server.getMetadata(), segment); - } - } - - private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) - { - SegmentId segmentId = segment.getId(); - synchronized (lock) { - log.debug("Adding segment[%s] for server[%s]", segment, server); - - SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - // servers escape the scope of this object so use ConcurrentSet - segmentLoadInfo = new SegmentLoadInfo(segment); - - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - if (timeline == null) { - timeline = new VersionedIntervalTimeline<>(Ordering.natural()); - timelines.put(segment.getDataSource(), timeline); - } - - timeline.add( - segment.getInterval(), - segment.getVersion(), - segment.getShardSpec().createChunk(segmentLoadInfo) - ); - segmentLoadInfos.put(segmentId, segmentLoadInfo); - } - segmentLoadInfo.addServer(server); - } - } - - private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) - { - SegmentId segmentId = segment.getId(); - - synchronized (lock) { - log.debug("Removing segment[%s] from server[%s].", segmentId, server); - - final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - log.warn("Told to remove non-existant segment[%s]", segmentId); - return; - } - segmentLoadInfo.removeServer(server); - if (segmentLoadInfo.isEmpty()) { - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - segmentLoadInfos.remove(segmentId); - - final PartitionChunk removedPartition = timeline.remove( - segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( - new SegmentLoadInfo( - segment - ) - ) - ); - - if (removedPartition == null) { - log.warn( - "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", - segment.getInterval(), - segment.getVersion() - ); - } - } - } - } - - @Override - public VersionedIntervalTimeline getTimeline(DataSource dataSource) - { - String table = Iterables.getOnlyElement(dataSource.getTableNames()); - synchronized (lock) { - return timelines.get(table); - } - } - - @Override - public Map getSegmentLoadInfos() - { - return segmentLoadInfos; - } - - @Override - public DruidServer getInventoryValue(String serverKey) - { - return baseView.getInventoryValue(serverKey); - } - - @Override - public Collection getInventory() - { - return baseView.getInventory(); - } - - @Override - public boolean isStarted() - { - return baseView.isStarted(); - } - - @Override - public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) - { - return baseView.isSegmentLoadedByServer(serverKey, segment); - } -} diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java new file mode 100644 index 000000000000..4bbfdf392892 --- /dev/null +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java @@ -0,0 +1,14 @@ +package org.apache.druid.client; + +import org.apache.druid.query.DataSource; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.VersionedIntervalTimeline; + +import java.util.Map; + +public interface CoordinatorServerViewInterface extends InventoryView +{ + + VersionedIntervalTimeline getTimeline(DataSource dataSource); + Map getSegmentLoadInfos(); +} diff --git a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java index a58c2e18957f..a4edb91a3e4c 100644 --- a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.client; import com.fasterxml.jackson.databind.ObjectMapper; @@ -24,18 +43,19 @@ import org.apache.druid.query.TableDataSource; import org.apache.druid.query.planning.DataSourceAnalysis; import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.TimelineLookup; import org.apache.druid.timeline.VersionedIntervalTimeline; import org.apache.druid.timeline.partition.PartitionChunk; -import org.apache.druid.utils.CollectionUtils; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; @@ -44,16 +64,21 @@ import java.util.function.Function; import java.util.stream.Collectors; +/** + * ServerView of coordinator for the state of segments being loaded in the cluster. + */ @ManageLifecycle -public class TimelineAwareCoordinatorServerView implements CoordinatorServerView, TimelineServerView +public class TimelineAwareCoordinatorServerView implements CoordinatorServerViewInterface, TimelineServerView { private static final Logger log = new Logger(TimelineAwareCoordinatorServerView.class); private final Object lock = new Object(); + private final Map segmentLoadInfos; private final ConcurrentMap clients = new ConcurrentHashMap<>(); private final Map selectors = new HashMap<>(); - private final Map> timelines = new HashMap<>(); + private final Map> timelines2 = new HashMap<>(); + private final Map> timelines; private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); @@ -85,6 +110,8 @@ public TimelineAwareCoordinatorServerView( this.baseView = baseView; this.segmentWatcherConfig = segmentWatcherConfig; this.emitter = emitter; + this.segmentLoadInfos = new HashMap<>(); + this.timelines = new HashMap<>(); this.tierSelectorStrategy = tierSelectorStrategy; this.warehouse = warehouse; this.queryWatcher = queryWatcher; @@ -106,6 +133,7 @@ public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSe @Override public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) { + serverRemovedSegment2(server, segment); serverRemovedSegment(server, segment); return ServerView.CallbackAction.CONTINUE; } @@ -153,45 +181,80 @@ public void start() throws InterruptedException private void removeServer(DruidServer server) { for (DataSegment segment : server.iterateAllSegments()) { + serverAddedSegment2(server.getMetadata(), segment); serverRemovedSegment(server.getMetadata(), segment); } } private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) { - final SegmentId segmentId = segment.getId(); + SegmentId segmentId = segment.getId(); synchronized (lock) { log.debug("Adding segment[%s] for server[%s]", segment, server); - ServerSelector selector = selectors.get(segmentId); - if (selector == null) { - selector = new ServerSelector(segment, tierSelectorStrategy); - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + // servers escape the scope of this object so use ConcurrentSet + segmentLoadInfo = new SegmentLoadInfo(segment); + + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); if (timeline == null) { - // broker needs to skip tombstones timeline = new VersionedIntervalTimeline<>(Ordering.natural()); timelines.put(segment.getDataSource(), timeline); } - timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); - selectors.put(segmentId, selector); + timeline.add( + segment.getInterval(), + segment.getVersion(), + segment.getShardSpec().createChunk(segmentLoadInfo) + ); + segmentLoadInfos.put(segmentId, segmentLoadInfo); } + segmentLoadInfo.addServer(server); + } + runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); + } - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); - if (inventoryValue == null) { - log.warn( - "Could not find server[%s] in inventory. Skipping addition of segment[%s].", - server.getName(), - segmentId - ); - return; - } else { - queryableDruidServer = addServer(inventoryValue); + private void serverAddedSegment2(final DruidServerMetadata server, final DataSegment segment) + { + final SegmentId segmentId = segment.getId(); + synchronized (lock) { + // in theory we could probably just filter this to ensure we don't put ourselves in here, to make broker tree + // query topologies, but for now just skip all brokers, so we don't create some sort of wild infinite query + // loop... + if (!server.getType().equals(ServerType.BROKER)) { + log.debug("Adding segment[%s] for server[%s]", segment, server); + ServerSelector selector = selectors.get(segmentId); + if (selector == null) { + selector = new ServerSelector(segment, tierSelectorStrategy); + + VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); + if (timeline == null) { + // broker needs to skip tombstones + timeline = new VersionedIntervalTimeline<>(Ordering.natural(), true); + timelines2.put(segment.getDataSource(), timeline); + } + + timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); + selectors.put(segmentId, selector); + } + + QueryableDruidServer queryableDruidServer = clients.get(server.getName()); + if (queryableDruidServer == null) { + DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); + if (inventoryValue == null) { + log.warn( + "Could not find server[%s] in inventory. Skipping addition of segment[%s].", + server.getName(), + segmentId + ); + return; + } else { + queryableDruidServer = addServer(inventoryValue); + } } + selector.addServerAndUpdateSegment(queryableDruidServer, segment); } - selector.addServerAndUpdateSegment(queryableDruidServer, segment); // run the callbacks, even if the segment came from a broker, lets downstream watchers decide what to do with it runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); } @@ -222,6 +285,46 @@ private DirectDruidClient makeDirectClient(DruidServer server) } private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) + { + SegmentId segmentId = segment.getId(); + + synchronized (lock) { + log.debug("Removing segment[%s] from server[%s].", segmentId, server); + + final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); + if (segmentLoadInfo == null) { + log.warn("Told to remove non-existant segment[%s]", segmentId); + return; + } + if (segmentLoadInfo.removeServer(server)) { + runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); + } + if (segmentLoadInfo.isEmpty()) { + VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + segmentLoadInfos.remove(segmentId); + + final PartitionChunk removedPartition = timeline.remove( + segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( + new SegmentLoadInfo( + segment + ) + ) + ); + + if (removedPartition == null) { + log.warn( + "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", + segment.getInterval(), + segment.getVersion() + ); + } else { + runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); + } + } + } + } + + private void serverRemovedSegment2(DruidServerMetadata server, DataSegment segment) { final SegmentId segmentId = segment.getId(); final ServerSelector selector; @@ -229,6 +332,8 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen synchronized (lock) { log.debug("Removing segment[%s] from server[%s].", segmentId, server); + // we don't store broker segments here, but still run the callbacks for the segment being removed from the server + // since the broker segments are not stored on the timeline, do not fire segmentRemoved event selector = selectors.get(segmentId); if (selector == null) { log.warn("Told to remove non-existant segment[%s]", segmentId); @@ -249,11 +354,11 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen segmentId ); } else { - runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); + // runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); } if (selector.isEmpty()) { - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); + VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); selectors.remove(segmentId); final PartitionChunk removedPartition = timeline.remove( @@ -267,7 +372,7 @@ private void serverRemovedSegment(DruidServerMetadata server, DataSegment segmen segment.getVersion() ); } else { - runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); + // runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); } } } @@ -278,14 +383,19 @@ public VersionedIntervalTimeline getTimeline(DataSource { String table = Iterables.getOnlyElement(dataSource.getTableNames()); synchronized (lock) { - timelines.get(table).getAllTimelineEntries() + // build a new timeline? } } @Override public Map getSegmentLoadInfos() { - return CollectionUtils.mapValues(selectors, ServerSelector::toSegmentLoadInfo); + // map + } + + public Set getLoadedSegmentIds() + { + return segmentLoadInfos.keySet(); } @Override @@ -351,7 +461,7 @@ public Optional> getTimeline(Da .orElseThrow(() -> new ISE("Cannot handle base datasource: %s", analysis.getBaseDataSource())); synchronized (lock) { - return Optional.ofNullable(timelines.get(table.getName())); + return Optional.ofNullable(timelines2.get(table.getName())); } } diff --git a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java index 583681bf7674..8bc97a7b6efa 100644 --- a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java @@ -19,8 +19,11 @@ package org.apache.druid.client.selector; +import com.google.common.collect.Sets; import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap; import org.apache.druid.client.DataSegmentInterner; +import org.apache.druid.client.ImmutableSegmentLoadInfo; +import org.apache.druid.client.SegmentLoadInfo; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunner; import org.apache.druid.server.coordination.DruidServerMetadata; @@ -216,4 +219,9 @@ public boolean hasData() return segment.get().hasData(); } + public SegmentLoadInfo toSegmentLoadInfo() + { + List allServers = getAllServers(); + return new SegmentLoadInfo(segment.get(), Sets.newConcurrentHashSet(allServers)); + } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java index 104194531769..cd2f849438ee 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java @@ -163,11 +163,9 @@ public AvailableSegmentMetadata build() public String toString() { return "AvailableSegmentMetadata{" + - "segment=" + segment + + "segmentId=" + segment.getId() + ", isRealtime=" + isRealtime + - ", segmentServers=" + segmentServers + ", numRows=" + numRows + - ", rowSignature=" + rowSignature + '}'; } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java b/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java index cad5cb882c62..9fb13fe5f0e5 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java @@ -6,6 +6,7 @@ public class DatasourceSchema { + // dsinfo private final String datasource; private final RowSignature rowSignature; diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 971c45cd3885..850f9746e7f2 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -281,6 +281,17 @@ public ServerView.CallbackAction serverSegmentRemoved( ); } + public void removeFromTable(String s) + { + tables.remove(s); + } + + public boolean tablesContains(String s) + { + return tables.containsKey(s); + } + + private void startCacheExec() { cacheExec.submit( @@ -523,7 +534,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } ); } - if (!tables.containsKey(segment.getDataSource())) { + if (!tablesContains(segment.getDataSource())) { refreshImmediately = true; } @@ -554,7 +565,7 @@ void removeSegment(final DataSegment segment) totalSegments--; } if (segmentsMap.isEmpty()) { - tables.remove(segment.getDataSource()); + removeFromTable(segment.getDataSource()); log.info("dataSource [%s] no longer exists, all metadata removed.", segment.getDataSource()); return null; } else { diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 9fa73970100f..70ad14f625e6 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -30,12 +30,12 @@ import com.sun.jersey.spi.container.ResourceFilters; import it.unimi.dsi.fastutil.objects.Object2LongMap; import org.apache.commons.lang.StringUtils; -import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.Alpha; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableSegmentLoadInfo; -import org.apache.druid.client.SegmentLoadInfo; +import org.apache.druid.client.selector.ServerSelector; import org.apache.druid.common.guava.FutureUtils; import org.apache.druid.guice.annotations.PublicApi; import org.apache.druid.java.util.common.DateTimes; @@ -102,7 +102,7 @@ public class DataSourcesResource private static final Logger log = new Logger(DataSourcesResource.class); private static final long DEFAULT_LOADSTATUS_INTERVAL_OFFSET = 14 * 24 * 60 * 60 * 1000; - private final CoordinatorServerView serverInventoryView; + private final Alpha serverInventoryView; private final SegmentsMetadataManager segmentsMetadataManager; private final MetadataRuleManager metadataRuleManager; private final OverlordClient overlordClient; @@ -111,7 +111,7 @@ public class DataSourcesResource @Inject public DataSourcesResource( - CoordinatorServerView serverInventoryView, + Alpha serverInventoryView, SegmentsMetadataManager segmentsMetadataManager, MetadataRuleManager metadataRuleManager, @Nullable OverlordClient overlordClient, @@ -820,7 +820,7 @@ public Response getServedSegmentsInInterval( @QueryParam("partial") final boolean partial ) { - TimelineLookup timeline = serverInventoryView.getTimeline( + TimelineLookup timeline = serverInventoryView.getTimeline( new TableDataSource(dataSourceName) ); final Interval theInterval = Intervals.of(interval.replace('_', '/')); @@ -833,19 +833,19 @@ public Response getServedSegmentsInInterval( } private Iterable prepareServedSegmentsInInterval( - TimelineLookup dataSourceServingTimeline, + TimelineLookup dataSourceServingTimeline, Interval interval ) { - Iterable> lookup = + Iterable> lookup = dataSourceServingTimeline.lookupWithIncompletePartitions(interval); return FunctionalIterable .create(lookup) .transformCat( - (TimelineObjectHolder input) -> + (TimelineObjectHolder input) -> Iterables.transform( input.getObject(), - (PartitionChunk chunk) -> chunk.getObject().toImmutableSegmentLoadInfo() + (PartitionChunk chunk) -> chunk.getObject().toImmutableSegmentLoadInfo() ) ); } @@ -885,7 +885,7 @@ public Response isHandOffComplete( return Response.ok(true).build(); } - TimelineLookup timeline = serverInventoryView.getTimeline( + TimelineLookup timeline = serverInventoryView.getTimeline( new TableDataSource(dataSourceName) ); if (timeline == null) { diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 9c9622e05e50..3f3060be6683 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -28,6 +28,7 @@ import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.Segments; +import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.segment.metadata.SegmentMetadataCache; @@ -42,6 +43,7 @@ import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.joda.time.Interval; +import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; @@ -70,6 +72,7 @@ @Path("/druid/coordinator/v1/metadata") public class MetadataResource { + private final Logger log = new Logger(MetadataResource.class); private final SegmentsMetadataManager segmentsMetadataManager; private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; @@ -145,7 +148,8 @@ public Response getDataSources( public Response getAllUsedSegments( @Context final HttpServletRequest req, @QueryParam("datasources") final @Nullable Set dataSources, - @QueryParam("includeOvershadowedStatus") final @Nullable String includeOvershadowedStatus + @QueryParam("includeOvershadowedStatus") final @Nullable String includeOvershadowedStatus, + @QueryParam("includeOvershadowedStatus") final @Nullable String includeRealtime ) { if (includeOvershadowedStatus != null) { @@ -188,45 +192,52 @@ private Response getAllUsedSegmentsWithAdditionalDetails( .filter(dataSourceWithUsedSegments -> dataSources.contains(dataSourceWithUsedSegments.getName())) .collect(Collectors.toList()); } - final Stream usedSegments = dataSourcesWithUsedSegments - .stream() - .flatMap(t -> t.getSegments().stream()); final Set overshadowedSegments = dataSourcesSnapshot.getOvershadowedSegments(); + final Set segmentAlreadySeen = new HashSet<>(); + final Stream segmentStatus = dataSourcesWithUsedSegments + .stream() + .flatMap(t -> t.getSegments().stream()) + .map(segment -> { + // The replication factor for unloaded segments is 0 as they will be unloaded soon + boolean isOvershadowed = overshadowedSegments.contains(segment); + AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata( + segment.getDataSource(), + segment.getId() + ); + Integer replicationFactor = isOvershadowed ? (Integer) 0 + : coordinator.getReplicationFactor(segment.getId()); + Long numRows = (null != availableSegmentMetadata) ? availableSegmentMetadata.getNumRows() : null; + segmentAlreadySeen.add(segment.getId()); + return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, 20L, true, 0L); + }).peek(v -> log.info("peeking into first stream segmentseen [%s], id [%s] isPublished [%s]", segmentAlreadySeen, v.getDataSegment().getId(), v.isPublished())); - Set segmentAlreadySeen = new HashSet<>(); - final Stream segmentStatus = usedSegments.map(segment -> { - // The replication factor for unloaded segments is 0 as they will be unloaded soon - boolean isOvershadowed = overshadowedSegments.contains(segment); - AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata(segment.getDataSource(), segment.getId()); - Integer replicationFactor = isOvershadowed ? (Integer) 0 - : coordinator.getReplicationFactor(segment.getId()); - Long numRows = (null != availableSegmentMetadata) ? availableSegmentMetadata.getNumRows() : null; - segmentAlreadySeen.add(segment.getId()); - return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, numRows, 0L, true); - }); + log.info("printing the content in smc cache %s", segmentMetadataCache.getSegmentMetadataSnapshot().values()); final Stream realtimeSegmentStatus = segmentMetadataCache .getSegmentMetadataSnapshot() .values() .stream() + .peek(v -> log.info("peeking into second stream (first) segmentseen [%s], id [%s]", segmentAlreadySeen, v.getSegment().getId())) .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment().getId())) .map(availableSegmentMetadata -> { return new SegmentStatusInCluster( availableSegmentMetadata.getSegment(), false, (int) availableSegmentMetadata.getNumReplicas(), - availableSegmentMetadata.getNumRows(), - availableSegmentMetadata.isRealtime(), - false + /**availableSegmentMetadata.getNumRows(), **/ 30L, + false, + availableSegmentMetadata.isRealtime() ); - }); + }).peek(v -> log.info("peeking into second stream (second) segmentseen [%s], id [%s]", segmentAlreadySeen, v.getDataSegment().getId()));; + + Stream combined = Stream.concat(segmentStatus, realtimeSegmentStatus).peek(v -> log.info("combined stream element %s", v)); final Function> raGenerator = segment -> Collections .singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getDataSegment().getDataSource())); final Iterable authorizedSegments = AuthorizationUtils.filterAuthorizedResources( req, - Stream.concat(segmentStatus, realtimeSegmentStatus)::iterator, + combined::iterator, raGenerator, authorizerMapper ); diff --git a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java similarity index 98% rename from server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java rename to server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java index 11a38af228b6..190a41e53f9e 100644 --- a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java +++ b/server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java @@ -52,7 +52,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; -public class CoordinatorServerViewTest extends CuratorTestBase +public class TimelineAwareCoordinatorServerViewTest extends CuratorTestBase { private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; @@ -63,9 +63,9 @@ public class CoordinatorServerViewTest extends CuratorTestBase private CountDownLatch segmentRemovedLatch; private BatchServerInventoryView baseView; - private CoordinatorServerViewImpl overlordServerView; + private TimelineAwareCoordinatorServerView overlordServerView; - public CoordinatorServerViewImplImplTest() + public TimelineAwareCoordinatorServerViewTest() { jsonMapper = TestHelper.makeJsonMapper(); zkPathsConfig = new ZkPathsConfig(); @@ -332,7 +332,7 @@ public CallbackAction segmentViewInitialized() } }; - overlordServerView = new CoordinatorServerViewImpl( + overlordServerView = new TimelineAwareCoordinatorServerView( baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter() diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java index b9b6fc8516fe..647eab9c6c45 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java @@ -30,7 +30,7 @@ import org.apache.curator.utils.ZKPaths; import org.apache.druid.client.BatchServerInventoryView; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; -import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.Alpha; import org.apache.druid.client.DataSourcesSnapshot; import org.apache.druid.client.DruidServer; import org.apache.druid.client.ImmutableDruidDataSource; @@ -97,7 +97,7 @@ public class CuratorDruidCoordinatorTest extends CuratorTestBase private static final long COORDINATOR_PERIOD = 100; private BatchServerInventoryView baseView; - private CoordinatorServerView serverView; + private Alpha serverView; private CountDownLatch segmentViewInitLatch; /** * The following two fields are changed during {@link #testMoveSegment()}, the change might not be visible from the @@ -405,7 +405,7 @@ public CallbackAction segmentViewInitialized() } }; - serverView = new CoordinatorServerView(baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter()); + serverView = new Alpha(baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter()); baseView.start(); diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 94b9bf8a84d0..5cdf1d5b73b9 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -28,7 +28,7 @@ import com.google.common.util.concurrent.Futures; import it.unimi.dsi.fastutil.objects.Object2LongMap; import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap; -import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.Alpha; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; import org.apache.druid.client.ImmutableDruidDataSource; @@ -83,7 +83,7 @@ public class DataSourcesResourceTest { - private CoordinatorServerView inventoryView; + private Alpha inventoryView; private DruidServer server; private List listDataSources; private List dataSegmentList; @@ -94,7 +94,7 @@ public class DataSourcesResourceTest public void setUp() { request = EasyMock.createStrictMock(HttpServletRequest.class); - inventoryView = EasyMock.createStrictMock(CoordinatorServerView.class); + inventoryView = EasyMock.createStrictMock(Alpha.class); server = EasyMock.niceMock(DruidServer.class); dataSegmentList = new ArrayList<>(); dataSegmentList.add( diff --git a/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java b/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java index f96d2a3e74c8..951070557d7d 100644 --- a/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.Alpha; import org.apache.druid.client.DruidServer; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; @@ -53,7 +53,7 @@ public void setUp() .build(); dummyServer.addDataSegment(segment); - CoordinatorServerView inventoryView = EasyMock.createMock(CoordinatorServerView.class); + Alpha inventoryView = EasyMock.createMock(Alpha.class); EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(dummyServer)).anyTimes(); EasyMock.expect(inventoryView.getInventoryValue(dummyServer.getName())).andReturn(dummyServer).anyTimes(); EasyMock.replay(inventoryView); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 93e7dad644e4..75475eac6f9b 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -40,7 +40,7 @@ import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.CachingClusteredClient; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; -import org.apache.druid.client.CoordinatorServerView; +import org.apache.druid.client.Alpha; import org.apache.druid.client.HttpServerInventoryViewResource; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; @@ -52,7 +52,6 @@ import org.apache.druid.guice.BrokerProcessingModule; import org.apache.druid.guice.ConditionalMultibind; import org.apache.druid.guice.ConfigProvider; -import org.apache.druid.guice.DruidProcessingModule; import org.apache.druid.guice.Jerseys; import org.apache.druid.guice.JoinableFactoryModule; import org.apache.druid.guice.JsonConfigProvider; @@ -257,11 +256,11 @@ public void configure(Binder binder) binder.bind(LookupCoordinatorManager.class).in(LazySingleton.class); binder.bind(CachingClusteredClient.class).in(LazySingleton.class); - binder.bind(CoordinatorServerView.class); - binder.bind(TimelineServerView.class).to(CoordinatorServerView.class).in(LazySingleton.class); + binder.bind(Alpha.class); + binder.bind(TimelineServerView.class).to(Alpha.class).in(LazySingleton.class); binder.bind(DruidCoordinator.class); - LifecycleModule.register(binder, CoordinatorServerView.class); + LifecycleModule.register(binder, Alpha.class); LifecycleModule.register(binder, MetadataStorage.class); LifecycleModule.register(binder, DruidCoordinator.class); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 0224841a6cbc..09f252cc3e20 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -1,5 +1,6 @@ package org.apache.druid.sql.calcite.schema; +import com.google.common.collect.ImmutableMap; import com.google.inject.Inject; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; @@ -13,6 +14,8 @@ import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.table.DatasourceTable; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -68,6 +71,24 @@ public void rebuildDatasource(String dataSource) } } + @Override + public Set getDatasourceNames() + { + return tables.keySet(); + } + + @Override + public void removeFromTable(String s) + { + tables.remove(s); + } + + @Override + public boolean tablesContains(String s) + { + return tables.containsKey(s); + } + public DatasourceTable.PhysicalDatasourceMetadata getPhysicalDatasourceMetadata(String name) { return tables.get(name); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index cf92bba0eb69..8b737ba9eb48 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -60,9 +60,13 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; + import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; + + + /** * This class polls the Coordinator in background to keep the latest published segments. * Provides {@link #getSegmentMetadata()} for others to get segments in metadata store. @@ -204,8 +208,8 @@ private ImmutableSortedSet fetchSegmentMetadata() segment.isOvershadowed(), replicationFactor, segment.getNumRows(), - segment.isRealtime(), - segment.isPublished() + true, + segment.isRealtime() ); builder.add(segmentStatusInCluster); } @@ -219,7 +223,8 @@ private void pollDatasourceSchema() Map physicalDatasourceMetadataMap = new HashMap<>(); - for (List partition : Iterables.partition(datasources, 10)) { + for (List partition : Iterables.partition(datasources, 100)) { + // retain watched datasources List datasourceSchemas = FutureUtils.getUnchecked(coordinatorClient.fetchDatasourceSchema( partition), true); @@ -246,6 +251,7 @@ ImmutableSortedSet getSegmentMetadata() protected SystemSchema.SegmentsTable.SegmentTableView getSegmentTableView() { ImmutableSortedSet allSegmentMetadata = getSegmentMetadata(); + log.info("logging polled segments from coordinator %s", allSegmentMetadata); final ImmutableSortedSet.Builder publishedSegmentBuilder = ImmutableSortedSet.naturalOrder(); Map availableSegmentMetadataMap; @@ -262,12 +268,14 @@ protected SystemSchema.SegmentsTable.SegmentTableView getSegmentTableView() { // build available segment metadata map by combining stuff from brokerServerView and numRows from data returned from coordinator availableSegmentMetadataMap = new HashMap<>(); Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); + // only look at watched ds, confirm if brokerServerView is also looking for watched ds for (SegmentStatusInCluster segmentStatusInCluster : allSegmentMetadata) { if (segmentStatusInCluster.isPublished()) { publishedSegmentBuilder.add(segmentStatusInCluster); } SegmentId segmentId = segmentStatusInCluster.getDataSegment().getId(); if (!brokerSegmentMetadata.containsKey(segmentId)) { + // log and count ignored segments continue; } ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index f1063c521ef0..73610534493a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -444,8 +444,9 @@ private Iterator> getAuthorizedAvaila protected static class SegmentTableView { + // pass the stitched private final Map availableSegmentMetadata; - private final Iterator publishedSegments; + private final Iterator publishedSegments; // coordinator private final int totalSegmentsCount; From a9ff64075fc81990390c361d0e0d678f292bd062 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 4 Sep 2023 13:55:50 +0530 Subject: [PATCH 05/36] Draft changes for the coordinator to conditionally build smc, and refactor on broker side --- .../apache/druid/client/BrokerServerView.java | 20 + ...ace.java => CoordinatorInventoryView.java} | 3 +- .../druid/client/CoordinatorServerView.java | 2 +- .../TimelineAwareCoordinatorServerView.java | 415 +----------------- .../client/coordinator/CoordinatorClient.java | 2 +- .../coordinator/CoordinatorClientImpl.java | 12 +- .../druid/client/selector/ServerSelector.java | 10 +- .../metadata/SegmentsMetadataManager.java | 5 - .../metadata/SegmentMetadataCacheConfig.java | 2 - .../server/http/DataSourcesResource.java | 24 +- .../org/apache/druid/cli/CliCoordinator.java | 63 ++- .../BrokerSegmentMetadataCacheConfig.java | 8 +- .../schema/BrokerSegmentMetadataView.java | 290 +++++++----- .../schema/DruidCalciteSchemaModule.java | 10 +- .../sql/calcite/schema/SystemSchema.java | 187 ++++---- 15 files changed, 388 insertions(+), 665 deletions(-) rename server/src/main/java/org/apache/druid/client/{CoordinatorServerViewInterface.java => CoordinatorInventoryView.java} (83%) diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index ee8490e1edf3..11624aad752a 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -440,4 +440,24 @@ public Set getDatasourceNames() { return timelines.keySet(); } + + Object getLock() + { + return lock; + } + + Map> getTimelines() + { + return timelines; + } + + Map getSelectors() + { + return selectors; + } + + ConcurrentMap getClients() + { + return clients; + } } diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java b/server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java similarity index 83% rename from server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java rename to server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java index 4bbfdf392892..a938c8864f1b 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerViewInterface.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java @@ -6,9 +6,8 @@ import java.util.Map; -public interface CoordinatorServerViewInterface extends InventoryView +public interface CoordinatorInventoryView extends InventoryView { - VersionedIntervalTimeline getTimeline(DataSource dataSource); Map getSegmentLoadInfos(); } diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index 6b6fecc2af6b..be4e90d9e600 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -26,7 +26,7 @@ * ServerView of coordinator for the state of segments being loaded in the cluster. */ @ManageLifecycle -public class CoordinatorServerView implements CoordinatorServerViewInterface +public class CoordinatorServerView implements CoordinatorInventoryView { private static final Logger log = new Logger(CoordinatorServerView.class); diff --git a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java index a4edb91a3e4c..88661386f0bc 100644 --- a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java @@ -21,79 +21,33 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterables; -import com.google.common.collect.Ordering; import com.google.inject.Inject; -import org.apache.druid.client.selector.QueryableDruidServer; import org.apache.druid.client.selector.ServerSelector; import org.apache.druid.client.selector.TierSelectorStrategy; import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.guice.annotations.EscalatedClient; import org.apache.druid.guice.annotations.Smile; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.query.DataSource; -import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.planning.DataSourceAnalysis; -import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.server.coordination.ServerType; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.TimelineLookup; import org.apache.druid.timeline.VersionedIntervalTimeline; -import org.apache.druid.timeline.partition.PartitionChunk; +import org.apache.druid.utils.CollectionUtils; import java.util.Collection; -import java.util.HashMap; -import java.util.List; +import java.util.Comparator; import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.function.Function; -import java.util.stream.Collectors; /** * ServerView of coordinator for the state of segments being loaded in the cluster. */ @ManageLifecycle -public class TimelineAwareCoordinatorServerView implements CoordinatorServerViewInterface, TimelineServerView +public class TimelineAwareCoordinatorServerView extends BrokerServerView implements CoordinatorInventoryView { - private static final Logger log = new Logger(TimelineAwareCoordinatorServerView.class); - - private final Object lock = new Object(); - - private final Map segmentLoadInfos; - private final ConcurrentMap clients = new ConcurrentHashMap<>(); - private final Map selectors = new HashMap<>(); - private final Map> timelines2 = new HashMap<>(); - private final Map> timelines; - - private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); - - private final ServerInventoryView baseView; - private final CoordinatorSegmentWatcherConfig segmentWatcherConfig; - - private final CountDownLatch initialized = new CountDownLatch(1); - private final ServiceEmitter emitter; - - private final TierSelectorStrategy tierSelectorStrategy; - - private final QueryToolChestWarehouse warehouse; - private final QueryWatcher queryWatcher; - private final ObjectMapper smileMapper; - private final HttpClient httpClient; + private final FilteredServerInventoryView baseView; @Inject public TimelineAwareCoordinatorServerView( @@ -101,301 +55,42 @@ public TimelineAwareCoordinatorServerView( final QueryWatcher queryWatcher, final @Smile ObjectMapper smileMapper, final @EscalatedClient HttpClient httpClient, - ServerInventoryView baseView, - CoordinatorSegmentWatcherConfig segmentWatcherConfig, + FilteredServerInventoryView baseView, + TierSelectorStrategy tierSelectorStrategy, ServiceEmitter emitter, - TierSelectorStrategy tierSelectorStrategy + CoordinatorSegmentWatcherConfig segmentWatcherConfig ) { - this.baseView = baseView; - this.segmentWatcherConfig = segmentWatcherConfig; - this.emitter = emitter; - this.segmentLoadInfos = new HashMap<>(); - this.timelines = new HashMap<>(); - this.tierSelectorStrategy = tierSelectorStrategy; - this.warehouse = warehouse; - this.queryWatcher = queryWatcher; - this.smileMapper = smileMapper; - this.httpClient = httpClient; - - ExecutorService exec = Execs.singleThreaded("CoordinatorServerView-%s"); - baseView.registerSegmentCallback( - exec, - new ServerView.SegmentCallback() - { - @Override - public ServerView.CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) - { - serverAddedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentRemoved(final DruidServerMetadata server, DataSegment segment) - { - serverRemovedSegment2(server, segment); - serverRemovedSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentViewInitialized() - { - initialized.countDown(); - runTimelineCallbacks(TimelineCallback::timelineInitialized); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - - baseView.registerServerRemovedCallback( - exec, - new ServerView.ServerRemovedCallback() - { - @Override - public ServerView.CallbackAction serverRemoved(DruidServer server) - { - removeServer(server); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - } - - @LifecycleStart - public void start() throws InterruptedException - { - if (segmentWatcherConfig.isAwaitInitializationOnStart()) { - final long startMillis = System.currentTimeMillis(); - log.info("%s waiting for initialization.", getClass().getSimpleName()); - initialized.await(); - final long endMillis = System.currentTimeMillis(); - log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().build( - "serverview/init/time", - endMillis - startMillis - )); - } - } - - private void removeServer(DruidServer server) - { - for (DataSegment segment : server.iterateAllSegments()) { - serverAddedSegment2(server.getMetadata(), segment); - serverRemovedSegment(server.getMetadata(), segment); - } - } - - private void serverAddedSegment(final DruidServerMetadata server, final DataSegment segment) - { - SegmentId segmentId = segment.getId(); - synchronized (lock) { - log.debug("Adding segment[%s] for server[%s]", segment, server); - - SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - // servers escape the scope of this object so use ConcurrentSet - segmentLoadInfo = new SegmentLoadInfo(segment); - - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - if (timeline == null) { - timeline = new VersionedIntervalTimeline<>(Ordering.natural()); - timelines.put(segment.getDataSource(), timeline); - } - - timeline.add( - segment.getInterval(), - segment.getVersion(), - segment.getShardSpec().createChunk(segmentLoadInfo) - ); - segmentLoadInfos.put(segmentId, segmentLoadInfo); - } - segmentLoadInfo.addServer(server); - } - runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); - } - - private void serverAddedSegment2(final DruidServerMetadata server, final DataSegment segment) - { - final SegmentId segmentId = segment.getId(); - synchronized (lock) { - // in theory we could probably just filter this to ensure we don't put ourselves in here, to make broker tree - // query topologies, but for now just skip all brokers, so we don't create some sort of wild infinite query - // loop... - if (!server.getType().equals(ServerType.BROKER)) { - log.debug("Adding segment[%s] for server[%s]", segment, server); - ServerSelector selector = selectors.get(segmentId); - if (selector == null) { - selector = new ServerSelector(segment, tierSelectorStrategy); - - VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); - if (timeline == null) { - // broker needs to skip tombstones - timeline = new VersionedIntervalTimeline<>(Ordering.natural(), true); - timelines2.put(segment.getDataSource(), timeline); - } - - timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector)); - selectors.put(segmentId, selector); - } - - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - DruidServer inventoryValue = baseView.getInventoryValue(server.getName()); - if (inventoryValue == null) { - log.warn( - "Could not find server[%s] in inventory. Skipping addition of segment[%s].", - server.getName(), - segmentId - ); - return; - } else { - queryableDruidServer = addServer(inventoryValue); - } - } - selector.addServerAndUpdateSegment(queryableDruidServer, segment); - } - // run the callbacks, even if the segment came from a broker, lets downstream watchers decide what to do with it - runTimelineCallbacks(callback -> callback.segmentAdded(server, segment)); - } - } - - private QueryableDruidServer addServer(DruidServer server) - { - QueryableDruidServer retVal = new QueryableDruidServer<>(server, makeDirectClient(server)); - QueryableDruidServer exists = clients.put(server.getName(), retVal); - if (exists != null) { - log.warn("QueryRunner for server[%s] already exists!? Well it's getting replaced", server); - } - - return retVal; - } - - private DirectDruidClient makeDirectClient(DruidServer server) - { - return new DirectDruidClient( - warehouse, - queryWatcher, - smileMapper, - httpClient, - server.getScheme(), - server.getHost(), - emitter - ); - } - - private void serverRemovedSegment(DruidServerMetadata server, DataSegment segment) - { - SegmentId segmentId = segment.getId(); - - synchronized (lock) { - log.debug("Removing segment[%s] from server[%s].", segmentId, server); - - final SegmentLoadInfo segmentLoadInfo = segmentLoadInfos.get(segmentId); - if (segmentLoadInfo == null) { - log.warn("Told to remove non-existant segment[%s]", segmentId); - return; - } - if (segmentLoadInfo.removeServer(server)) { - runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); - } - if (segmentLoadInfo.isEmpty()) { - VersionedIntervalTimeline timeline = timelines.get(segment.getDataSource()); - segmentLoadInfos.remove(segmentId); - - final PartitionChunk removedPartition = timeline.remove( - segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk( - new SegmentLoadInfo( - segment - ) - ) - ); - - if (removedPartition == null) { - log.warn( - "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", - segment.getInterval(), - segment.getVersion() - ); - } else { - runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); - } - } - } - } - - private void serverRemovedSegment2(DruidServerMetadata server, DataSegment segment) - { - final SegmentId segmentId = segment.getId(); - final ServerSelector selector; - - synchronized (lock) { - log.debug("Removing segment[%s] from server[%s].", segmentId, server); - - // we don't store broker segments here, but still run the callbacks for the segment being removed from the server - // since the broker segments are not stored on the timeline, do not fire segmentRemoved event - selector = selectors.get(segmentId); - if (selector == null) { - log.warn("Told to remove non-existant segment[%s]", segmentId); - return; - } - - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - log.warn( - "Could not find server[%s] in inventory. Skipping removal of segment[%s].", - server.getName(), - segmentId - ); - } else if (!selector.removeServer(queryableDruidServer)) { - log.warn( - "Asked to disassociate non-existant association between server[%s] and segment[%s]", - server, - segmentId - ); - } else { - // runTimelineCallbacks(callback -> callback.serverSegmentRemoved(server, segment)); - } - - if (selector.isEmpty()) { - VersionedIntervalTimeline timeline = timelines2.get(segment.getDataSource()); - selectors.remove(segmentId); - - final PartitionChunk removedPartition = timeline.remove( - segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector) - ); - - if (removedPartition == null) { - log.warn( - "Asked to remove timeline entry[interval: %s, version: %s] that doesn't exist", - segment.getInterval(), - segment.getVersion() - ); - } else { - // runTimelineCallbacks(callback -> callback.segmentRemoved(segment)); - } + super(warehouse, queryWatcher, smileMapper, httpClient, baseView, tierSelectorStrategy, emitter, new BrokerSegmentWatcherConfig() { + @Override + public boolean isAwaitInitializationOnStart() + { + return segmentWatcherConfig.isAwaitInitializationOnStart(); } - } + }); + this.baseView = baseView; } @Override public VersionedIntervalTimeline getTimeline(DataSource dataSource) { String table = Iterables.getOnlyElement(dataSource.getTableNames()); - synchronized (lock) { + synchronized (getLock()) { // build a new timeline? + VersionedIntervalTimeline timeline = getTimelines().get(table); + Collection x = timeline.iterateAllObjects(); + VersionedIntervalTimeline newTimeline = new VersionedIntervalTimeline<>(Comparator.naturalOrder()); + newTimeline.addAll(x.stream().map(v -> new VersionedIntervalTimeline.PartitionChunkEntry( + v.getSegment().getInterval(), v.getSegment().getVersion(), v.getSegment().getShardSpec().createChunk(v.toSegmentLoadInfo()))).iterator()); + + return newTimeline; } } @Override public Map getSegmentLoadInfos() { - // map - } - - public Set getLoadedSegmentIds() - { - return segmentLoadInfos.keySet(); + return CollectionUtils.mapValues(getSelectors(), ServerSelector::toSegmentLoadInfo); } @Override @@ -421,68 +116,4 @@ public boolean isSegmentLoadedByServer(String serverKey, DataSegment segment) { return baseView.isSegmentLoadedByServer(serverKey, segment); } - - private void runTimelineCallbacks(final Function function) - { - for (Map.Entry entry : timelineCallbacks.entrySet()) { - entry.getValue().execute( - () -> { - if (CallbackAction.UNREGISTER == function.apply(entry.getKey())) { - timelineCallbacks.remove(entry.getKey()); - } - } - ); - } - } - - @Override - public void registerTimelineCallback(Executor exec, TimelineCallback callback) - { - timelineCallbacks.put(callback, exec); - } - - @Override - public void registerServerRemovedCallback(Executor exec, ServerRemovedCallback callback) - { - throw new UnsupportedOperationException(); - } - - @Override - public void registerSegmentCallback(Executor exec, SegmentCallback callback) - { - baseView.registerSegmentCallback(exec, callback); - } - - @Override - public Optional> getTimeline(DataSourceAnalysis analysis) - { - final TableDataSource table = - analysis.getBaseTableDataSource() - .orElseThrow(() -> new ISE("Cannot handle base datasource: %s", analysis.getBaseDataSource())); - - synchronized (lock) { - return Optional.ofNullable(timelines2.get(table.getName())); - } - } - - @Override - public List getDruidServers() - { - return clients.values().stream() - .map(queryableDruidServer -> queryableDruidServer.getServer().toImmutableDruidServer()) - .collect(Collectors.toList()); - } - - @Override - public QueryRunner getQueryRunner(DruidServer server) - { - synchronized (lock) { - QueryableDruidServer queryableDruidServer = clients.get(server.getName()); - if (queryableDruidServer == null) { - log.error("No QueryRunner found for server name[%s].", server.getName()); - return null; - } - return queryableDruidServer.getQueryRunner(); - } - } } diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index 812f35d4622e..a56363afee83 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -48,7 +48,7 @@ public interface CoordinatorClient */ ListenableFuture> fetchUsedSegments(String dataSource, List intervals); - ListenableFuture> fetchDatasourceSchema(List datasources); + ListenableFuture> fetchDatasourceSchema(Set datasources); /** * Returns a new instance backed by a ServiceClient which follows the provided retryPolicy diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index dae00c73bea0..2a9e1891f086 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -26,17 +26,16 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.http.client.response.BytesFullResponseHandler; -import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.RequestBuilder; import org.apache.druid.rpc.ServiceClient; import org.apache.druid.rpc.ServiceRetryPolicy; +import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentStatusInCluster; import org.jboss.netty.handler.codec.http.HttpMethod; import org.joda.time.Interval; -import java.util.Iterator; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -111,10 +110,13 @@ public ListenableFuture> fetchUsedSegments(String dataSource, } @Override - public ListenableFuture> fetchDatasourceSchema(List datasources) + public ListenableFuture> fetchDatasourceSchema(Set datasources) { final String path = "/druid/coordinator/v1/metadata/datasourceSchema"; - + if (null == datasources) + { + datasources = new HashSet<>(); + } return FutureUtils.transform( client.asyncRequest( new RequestBuilder(HttpMethod.POST, path) diff --git a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java index 8bc97a7b6efa..6879600878e2 100644 --- a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java @@ -19,10 +19,8 @@ package org.apache.druid.client.selector; -import com.google.common.collect.Sets; import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap; import org.apache.druid.client.DataSegmentInterner; -import org.apache.druid.client.ImmutableSegmentLoadInfo; import org.apache.druid.client.SegmentLoadInfo; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunner; @@ -222,6 +220,12 @@ public boolean hasData() public SegmentLoadInfo toSegmentLoadInfo() { List allServers = getAllServers(); - return new SegmentLoadInfo(segment.get(), Sets.newConcurrentHashSet(allServers)); + SegmentLoadInfo segmentLoadInfo = new SegmentLoadInfo(segment.get()); + + for (DruidServerMetadata druidServerMetadata : allServers) { + segmentLoadInfo.addServer(druidServerMetadata); + } + + return segmentLoadInfo; } } diff --git a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java index 0b1468d0f129..0519445debfd 100644 --- a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java +++ b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java @@ -33,11 +33,6 @@ import java.util.List; import java.util.Set; -/** - * The difference between this class and org.apache.druid.sql.calcite.schema.MetadataSegmentView is that this class - * resides in Coordinator's memory, while org.apache.druid.sql.calcite.schema.MetadataSegmentView resides in Broker's - * memory. - */ public interface SegmentsMetadataManager { void startPollingDatabasePeriodically(); diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index c59ee23a77b2..c5d22c087794 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -7,10 +7,8 @@ public class SegmentMetadataCacheConfig { @JsonProperty private boolean awaitInitializationOnStart = true; - @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); - @JsonProperty private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 70ad14f625e6..ea92c4e8471a 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -30,12 +30,12 @@ import com.sun.jersey.spi.container.ResourceFilters; import it.unimi.dsi.fastutil.objects.Object2LongMap; import org.apache.commons.lang.StringUtils; -import org.apache.druid.client.Alpha; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; +import org.apache.druid.client.CoordinatorInventoryView; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableSegmentLoadInfo; -import org.apache.druid.client.selector.ServerSelector; +import org.apache.druid.client.SegmentLoadInfo; import org.apache.druid.common.guava.FutureUtils; import org.apache.druid.guice.annotations.PublicApi; import org.apache.druid.java.util.common.DateTimes; @@ -102,7 +102,7 @@ public class DataSourcesResource private static final Logger log = new Logger(DataSourcesResource.class); private static final long DEFAULT_LOADSTATUS_INTERVAL_OFFSET = 14 * 24 * 60 * 60 * 1000; - private final Alpha serverInventoryView; + private final CoordinatorInventoryView serverInventoryView; private final SegmentsMetadataManager segmentsMetadataManager; private final MetadataRuleManager metadataRuleManager; private final OverlordClient overlordClient; @@ -111,7 +111,7 @@ public class DataSourcesResource @Inject public DataSourcesResource( - Alpha serverInventoryView, + CoordinatorInventoryView serverInventoryView, SegmentsMetadataManager segmentsMetadataManager, MetadataRuleManager metadataRuleManager, @Nullable OverlordClient overlordClient, @@ -487,13 +487,13 @@ public Response getDatasourceLoadstatus( private SegmentsLoadStatistics computeSegmentLoadStatistics(Iterable segments) { - Set loadedSegmentIds = serverInventoryView.getLoadedSegmentIds(); + Map segmentLoadInfos = serverInventoryView.getSegmentLoadInfos(); int numPublishedSegments = 0; int numUnavailableSegments = 0; int numLoadedSegments = 0; for (DataSegment segment : segments) { numPublishedSegments++; - if (!loadedSegmentIds.contains(segment.getId())) { + if (!segmentLoadInfos.containsKey(segment.getId())) { numUnavailableSegments++; } else { numLoadedSegments++; @@ -820,7 +820,7 @@ public Response getServedSegmentsInInterval( @QueryParam("partial") final boolean partial ) { - TimelineLookup timeline = serverInventoryView.getTimeline( + TimelineLookup timeline = serverInventoryView.getTimeline( new TableDataSource(dataSourceName) ); final Interval theInterval = Intervals.of(interval.replace('_', '/')); @@ -833,19 +833,19 @@ public Response getServedSegmentsInInterval( } private Iterable prepareServedSegmentsInInterval( - TimelineLookup dataSourceServingTimeline, + TimelineLookup dataSourceServingTimeline, Interval interval ) { - Iterable> lookup = + Iterable> lookup = dataSourceServingTimeline.lookupWithIncompletePartitions(interval); return FunctionalIterable .create(lookup) .transformCat( - (TimelineObjectHolder input) -> + (TimelineObjectHolder input) -> Iterables.transform( input.getObject(), - (PartitionChunk chunk) -> chunk.getObject().toImmutableSegmentLoadInfo() + (PartitionChunk chunk) -> chunk.getObject().toImmutableSegmentLoadInfo() ) ); } @@ -885,7 +885,7 @@ public Response isHandOffComplete( return Response.ok(true).build(); } - TimelineLookup timeline = serverInventoryView.getTimeline( + TimelineLookup timeline = serverInventoryView.getTimeline( new TableDataSource(dataSourceName) ); if (timeline == null) { diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 75475eac6f9b..85c312f2b04e 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -37,12 +37,13 @@ import com.google.inject.util.Providers; import org.apache.curator.framework.CuratorFramework; import org.apache.druid.audit.AuditManager; -import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.CachingClusteredClient; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; -import org.apache.druid.client.Alpha; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.HttpServerInventoryViewResource; +import org.apache.druid.client.CoordinatorInventoryView; import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.TimelineAwareCoordinatorServerView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.Coordinator; import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig; @@ -192,12 +193,6 @@ protected List getModules() List modules = new ArrayList<>(); modules.add(JettyHttpClientModule.global()); - modules.add(new LegacyBrokerParallelMergeConfigModule()); - modules.add(new QueryRunnerFactoryModule()); - modules.add(new SegmentWranglerModule()); - modules.add(new QueryableModule()); - modules.add(new BrokerProcessingModule()); - modules.add(new JoinableFactoryModule()); modules.add( new Module() @@ -215,8 +210,6 @@ public void configure(Binder binder) binder.bind(MetadataStorage.class).toProvider(MetadataStorageProvider.class); - binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); - JsonConfigProvider.bind(binder, SegmentsMetadataManagerConfig.CONFIG_PREFIX, SegmentsMetadataManagerConfig.class); JsonConfigProvider.bind(binder, "druid.manager.rules", MetadataRuleManagerConfig.class); JsonConfigProvider.bind(binder, "druid.manager.lookups", LookupCoordinatorManagerConfig.class); @@ -227,13 +220,6 @@ public void configure(Binder binder) "druid.coordinator.balancer.cachingCost", CachingCostBalancerStrategyConfig.class ); - JsonConfigProvider.bind(binder, "druid.coordinator.segment", SegmentMetadataCacheConfig.class); - JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.broker.retryPolicy", RetryQueryRunnerConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.segment", BrokerSegmentWatcherConfig.class); binder.bind(RedirectFilter.class).in(LazySingleton.class); if (beOverlord) { @@ -242,6 +228,13 @@ public void configure(Binder binder) binder.bind(RedirectInfo.class).to(CoordinatorRedirectInfo.class).in(LazySingleton.class); } + if (isSegmentMetadataCacheEnabled()) { + binder.install(new SegmentMetadataCacheModule()); + } else { + binder.bind(CoordinatorInventoryView.class).to(CoordinatorServerView.class).in(LazySingleton.class); + LifecycleModule.register(binder, CoordinatorServerView.class); + } + binder.bind(SegmentsMetadataManager.class) .toProvider(SegmentsMetadataManagerProvider.class) .in(ManageLifecycle.class); @@ -255,17 +248,12 @@ public void configure(Binder binder) .in(ManageLifecycle.class); binder.bind(LookupCoordinatorManager.class).in(LazySingleton.class); - binder.bind(CachingClusteredClient.class).in(LazySingleton.class); - binder.bind(Alpha.class); - binder.bind(TimelineServerView.class).to(Alpha.class).in(LazySingleton.class); + binder.bind(DruidCoordinator.class); - LifecycleModule.register(binder, Alpha.class); LifecycleModule.register(binder, MetadataStorage.class); LifecycleModule.register(binder, DruidCoordinator.class); - LifecycleModule.register(binder, SegmentMetadataCache.class); - binder.bind(JettyServerInitializer.class) .to(CoordinatorJettyServerInitializer.class); @@ -530,4 +518,33 @@ public Supplier> get() }; } } + + private static class SegmentMetadataCacheModule implements Module + { + @Override + public void configure(Binder binder) + { + binder.install(new LegacyBrokerParallelMergeConfigModule()); + binder.install(new QueryRunnerFactoryModule()); + binder.install(new SegmentWranglerModule()); + binder.install(new QueryableModule()); + binder.install(new BrokerProcessingModule()); + binder.install(new JoinableFactoryModule()); + + JsonConfigProvider.bind(binder, "druid.coordinator.segment", SegmentMetadataCacheConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); + JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.broker.retryPolicy", RetryQueryRunnerConfig.class); + + binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); + binder.bind(CachingClusteredClient.class).in(LazySingleton.class); + binder.bind(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); + binder.bind(CoordinatorInventoryView.class).to(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); + binder.bind(TimelineServerView.class).to(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); + LifecycleModule.register(binder, TimelineAwareCoordinatorServerView.class); + LifecycleModule.register(binder, SegmentMetadataCache.class); + } + } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index d7a059710893..9219706940fe 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -6,13 +6,13 @@ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig { @JsonProperty - private boolean metadataSegmentCacheEnable = false; + private boolean metadataSegmentCacheEnable = true; @JsonProperty private long metadataSegmentPollPeriod = 60000; @JsonProperty - private boolean useSegmentMetadataCache = false; + private boolean segmentMetadataCacheEnabled = false; public boolean isMetadataSegmentCacheEnable() { @@ -24,8 +24,8 @@ public long getMetadataSegmentPollPeriod() return metadataSegmentPollPeriod; } - public boolean isUseSegmentMetadataCache() + public boolean isSegmentMetadataCacheEnabled() { - return useSegmentMetadataCache; + return segmentMetadataCacheEnabled; } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index 8b737ba9eb48..bb1d4f4ec6ff 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -25,8 +25,6 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableSortedSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.Uninterruptibles; import com.google.inject.Inject; import org.apache.druid.client.BrokerSegmentWatcherConfig; @@ -45,35 +43,32 @@ import org.apache.druid.java.util.common.lifecycle.LifecycleStart; import org.apache.druid.java.util.common.lifecycle.LifecycleStop; import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.metadata.DatasourceSchema; -import org.apache.druid.metadata.SegmentsMetadataManager; +import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable.SegmentTableView; import org.apache.druid.sql.calcite.table.DatasourceTable; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; - import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; - - - /** * This class polls the Coordinator in background to keep the latest published segments. * Provides {@link #getSegmentMetadata()} for others to get segments in metadata store. * - * The difference between this class and {@link SegmentsMetadataManager} is that this class resides - * in Broker's memory, while {@link SegmentsMetadataManager} resides in Coordinator's memory. In - * fact, this class polls the data from {@link SegmentsMetadataManager} object in the memory of the + * This class polls the data from {@link SegmentsMetadataManager} object in the memory of the * currently leading Coordinator via HTTP queries. */ @ManageLifecycle @@ -92,11 +87,16 @@ public class BrokerSegmentMetadataView private final boolean useSegmentMetadataCache; + private final boolean includeRealtimeSegments; + + private final boolean pollDsSchema; + /** * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and * sys.segments queries return the segments in sorted order based on segmentId. * - * Volatile since this reference is reassigned in {@code poll()} and then read in {@code getPublishedSegments()} + * Volatile since this reference is reassigned in {@code pollSegmentMetadata()} + * and then read in {@code getSegmentMetadata()} * from other threads. */ @MonotonicNonNull @@ -129,21 +129,25 @@ public BrokerSegmentMetadataView( final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder, final BrokerServerView brokerServerView, final BrokerSegmentMetadataCache segmentMetadataCache - ) + ) { - Preconditions.checkNotNull(config, "SegmentMetadataCacheConfig"); + Preconditions.checkNotNull(config, "BrokerSegmentMetadataCacheConfig"); this.druidLeaderClient = druidLeaderClient; this.objectMapper = objectMapper; this.coordinatorClient = coordinatorClient; this.segmentWatcherConfig = segmentWatcherConfig; + this.isMetadataSegmentCacheEnabled = config.isMetadataSegmentCacheEnable(); + this.useSegmentMetadataCache = config.isSegmentMetadataCacheEnabled(); + this.includeRealtimeSegments = !useSegmentMetadataCache; + this.pollDsSchema = !useSegmentMetadataCache; + this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); - this.scheduledExec = Execs.scheduledSingleThreaded("MetadataSegmentView-Cache--%d"); + this.scheduledExec = Execs.scheduledSingleThreaded("SegmentMetadataView-Cache--%d"); this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() .expireAfterAccess(10, TimeUnit.MINUTES) .build(); this.physicalDatasourceMetadataBuilder = physicalDatasourceMetadataBuilder; - this.useSegmentMetadataCache = config.isUseSegmentMetadataCache(); this.brokerServerView = brokerServerView; this.segmentMetadataCache = segmentMetadataCache; } @@ -155,7 +159,7 @@ public void start() throw new ISE("can't start."); } try { - if (isMetadataSegmentCacheEnabled || !useSegmentMetadataCache) { + if (isMetadataSegmentCacheEnabled || pollDsSchema) { scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); } lifecycleLock.started(); @@ -174,70 +178,158 @@ public void stop() throw new ISE("can't stop."); } log.info("MetadataSegmentView is stopping."); - if (isMetadataSegmentCacheEnabled) { + if (isMetadataSegmentCacheEnabled || pollDsSchema) { scheduledExec.shutdown(); } log.info("MetadataSegmentView Stopped."); } - private void pollSegmentMetadata() + protected DatasourceTable.PhysicalDatasourceMetadata getDatasource(String name) { - log.info("Polling segment metadata from coordinator"); + if (useSegmentMetadataCache) { + return segmentMetadataCache.getPhysicalDatasourceMetadata(name); + } + return datasourceSchemaMap.get(name); + } - segmentMetadata = fetchSegmentMetadata(); - segmentMetadataCachePopulated.countDown(); + protected Set getDatasourceNames() + { + if (useSegmentMetadataCache) { + return segmentMetadataCache.getDatasourceNames(); + } + return datasourceSchemaMap.keySet(); } - private ImmutableSortedSet fetchSegmentMetadata() + protected Iterator getSegmentTableView() { + if (useSegmentMetadataCache) { + return getSegmentTableViewFromCoordinatorAndSmc(); + } + return getSegmentTableViewFromCoordinator(); + } + + private Iterator getSegmentTableViewFromCoordinatorAndSmc() { - final Iterator metadataSegments = - querySegmentMetadata(segmentWatcherConfig.getWatchedDataSources()); + final ImmutableSortedSet publishedSegments = getSegmentMetadata(); + final Map availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); - final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); - while (metadataSegments.hasNext()) { - final SegmentStatusInCluster segment = metadataSegments.next(); - final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); - Integer replicationFactor = segment.getReplicationFactor(); - if (replicationFactor == null) { - replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); - } else { - segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); + final List segmentsTableView = new ArrayList<>(); + + Set seenSegments = new HashSet<>(); + for (SegmentStatusInCluster segmentStatusInCluster : publishedSegments) + { + DataSegment segment = segmentStatusInCluster.getDataSegment(); + SegmentId segmentId = segment.getId(); + AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataMap.get(segmentId); + + long numReplicas = 0L, numRows = 0L, isRealtime = 0L, isAvailable = 0L; + if (availableSegmentMetadata != null) { + numReplicas = availableSegmentMetadata.getNumReplicas(); + numRows = availableSegmentMetadata.getNumRows(); + isAvailable = 1L; + isRealtime = availableSegmentMetadata.isRealtime(); } - final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( - interned, - segment.isOvershadowed(), - replicationFactor, - segment.getNumRows(), - true, - segment.isRealtime() + + SegmentTableView segmentTableView = new SegmentTableView( + segment, + isAvailable, + isRealtime, + numReplicas, + numRows, + segmentStatusInCluster.getReplicationFactor(), + segmentStatusInCluster.isOvershadowed(), + true ); - builder.add(segmentStatusInCluster); + seenSegments.add(segmentId); + segmentsTableView.add((segmentTableView)); } - return builder.build(); + + for (Map.Entry availableSegmentMetadataEntry : availableSegmentMetadataMap.entrySet()) + { + if (seenSegments.contains(availableSegmentMetadataEntry.getKey())) { + continue; + } + AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataEntry.getValue(); + SegmentTableView segmentTableView = new SegmentTableView( + availableSegmentMetadata.getSegment(), + 1L, + availableSegmentMetadata.isRealtime(), + availableSegmentMetadata.getNumReplicas(), + availableSegmentMetadata.getNumRows(), + null, + false, + false + ); + segmentsTableView.add(segmentTableView); + } + + return segmentsTableView.iterator(); + } + + private Iterator getSegmentTableViewFromCoordinator() + { + final ImmutableSortedSet allSegments = getSegmentMetadata(); + final Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); + + final List segmentsTableView = new ArrayList<>(); + + for (SegmentStatusInCluster segmentStatusInCluster : allSegments) { + SegmentId segmentId = segmentStatusInCluster.getDataSegment().getId(); + ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); + long numReplicas = 0L, isAvailable = 0L, numRows = 0L; + if (null != serverSelector) { + numReplicas = serverSelector.getAllServers().size(); + isAvailable = 1L; + } + if (null != segmentStatusInCluster.getNumRows()) + { + numRows = segmentStatusInCluster.getNumRows(); + } + + SegmentTableView segmentTableView = new SegmentTableView( + segmentStatusInCluster.getDataSegment(), + isAvailable, + segmentStatusInCluster.isRealtime(), + numReplicas, + numRows, + segmentStatusInCluster.getReplicationFactor(), + segmentStatusInCluster.isOvershadowed(), + segmentStatusInCluster.isPublished() + ); + segmentsTableView.add(segmentTableView); + } + + return segmentsTableView.iterator(); } private void pollDatasourceSchema() { log.info("Polling datasource schema from coordinator."); - Set datasources = useSegmentMetadataCache ? segmentMetadataCache.getDatasourceNames() : brokerServerView.getDatasourceNames(); + + Set watchedDatasources = segmentWatcherConfig.getWatchedDataSources(); Map physicalDatasourceMetadataMap = new HashMap<>(); - for (List partition : Iterables.partition(datasources, 100)) { - // retain watched datasources - List datasourceSchemas = FutureUtils.getUnchecked(coordinatorClient.fetchDatasourceSchema( - partition), true); + List datasourceSchemas = FutureUtils.getUnchecked( + coordinatorClient.fetchDatasourceSchema(watchedDatasources), true); - for (DatasourceSchema datasourceSchema : datasourceSchemas) { - physicalDatasourceMetadataMap.put( - datasourceSchema.getDatasource(), - physicalDatasourceMetadataBuilder.build(datasourceSchema.getDatasource(), datasourceSchema.getRowSignature())); - } + for (DatasourceSchema datasourceSchema : datasourceSchemas) { + physicalDatasourceMetadataMap.put( + datasourceSchema.getDatasource(), + physicalDatasourceMetadataBuilder.build(datasourceSchema.getDatasource(), datasourceSchema.getRowSignature()) + ); } this.datasourceSchemaMap = physicalDatasourceMetadataMap; } + private void pollSegmentMetadata() + { + log.info("Polling segment metadata from coordinator"); + + segmentMetadata = fetchSegmentMetadata(); + segmentMetadataCachePopulated.countDown(); + } + ImmutableSortedSet getSegmentMetadata() { if (isMetadataSegmentCacheEnabled) { @@ -248,65 +340,32 @@ ImmutableSortedSet getSegmentMetadata() } } - protected SystemSchema.SegmentsTable.SegmentTableView getSegmentTableView() { - ImmutableSortedSet allSegmentMetadata = getSegmentMetadata(); - - log.info("logging polled segments from coordinator %s", allSegmentMetadata); - final ImmutableSortedSet.Builder publishedSegmentBuilder = ImmutableSortedSet.naturalOrder(); - - Map availableSegmentMetadataMap; - - if (useSegmentMetadataCache) { - availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); - - for (SegmentStatusInCluster segmentStatusInCluster : allSegmentMetadata) { - if (segmentStatusInCluster.isPublished()) { - publishedSegmentBuilder.add(segmentStatusInCluster); - } - } - } else { - // build available segment metadata map by combining stuff from brokerServerView and numRows from data returned from coordinator - availableSegmentMetadataMap = new HashMap<>(); - Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); - // only look at watched ds, confirm if brokerServerView is also looking for watched ds - for (SegmentStatusInCluster segmentStatusInCluster : allSegmentMetadata) { - if (segmentStatusInCluster.isPublished()) { - publishedSegmentBuilder.add(segmentStatusInCluster); - } - SegmentId segmentId = segmentStatusInCluster.getDataSegment().getId(); - if (!brokerSegmentMetadata.containsKey(segmentId)) { - // log and count ignored segments - continue; - } - ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); - - AvailableSegmentMetadata availableSegmentMetadata = - AvailableSegmentMetadata.builder( - segmentStatusInCluster.getDataSegment(), - segmentStatusInCluster.isRealtime(), - Sets.newHashSet(serverSelector.getAllServers()), - null, - segmentStatusInCluster.getNumRows() == null ? -1 : segmentStatusInCluster.getNumRows() - ) - .build(); - - availableSegmentMetadataMap.put(segmentId, availableSegmentMetadata); + private ImmutableSortedSet fetchSegmentMetadata() + { + final Iterator metadataSegments = + querySegmentMetadata(segmentWatcherConfig.getWatchedDataSources()); + final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); + while (metadataSegments.hasNext()) { + final SegmentStatusInCluster segment = metadataSegments.next(); + final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); + Integer replicationFactor = segment.getReplicationFactor(); + if (replicationFactor == null) { + replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); + } else { + segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); } + final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( + interned, + segment.isOvershadowed(), + replicationFactor, + segment.getNumRows(), + true, + segment.isRealtime() + ); + builder.add(segmentStatusInCluster); } - - log.info("Logging Segment table view. availableSmMap [%s], published segments [%s]", availableSegmentMetadataMap, publishedSegmentBuilder); - return new SystemSchema.SegmentsTable.SegmentTableView(availableSegmentMetadataMap, publishedSegmentBuilder.build()); - } - - protected DatasourceTable.PhysicalDatasourceMetadata getDatasource(String name) - { - return useSegmentMetadataCache ? segmentMetadataCache.getPhysicalDatasourceMetadata(name) : datasourceSchemaMap.get(name); - } - - protected Set getDatasourceNames() - { - return useSegmentMetadataCache ? segmentMetadataCache.getDatasourceNames() : datasourceSchemaMap.keySet(); + return builder.build(); } // Note that coordinator must be up to get segments @@ -314,20 +373,21 @@ private JsonParserIterator querySegmentMetadata( Set watchedDataSources ) { - String query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"; + final StringBuilder queryBuilder = new StringBuilder("/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"); + if (includeRealtimeSegments) { + queryBuilder.append("&includeRealtimeSegments"); + } if (watchedDataSources != null && !watchedDataSources.isEmpty()) { log.debug( "filtering datasources in published segments based on broker's watchedDataSources[%s]", watchedDataSources); - final StringBuilder sb = new StringBuilder(); for (String ds : watchedDataSources) { - sb.append("datasources=").append(ds).append("&"); + queryBuilder.append("&datasources=").append(ds).append("&"); } - sb.setLength(sb.length() - 1); - query = "/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&" + sb; + queryBuilder.setLength(queryBuilder.length() - 1); } return SystemSchema.getThingsFromLeaderNode( - query, + queryBuilder.toString(), new TypeReference() { }, @@ -347,7 +407,7 @@ public void run() if (isMetadataSegmentCacheEnabled) { pollSegmentMetadata(); } - if (!useSegmentMetadataCache) { + if (pollDsSchema) { pollDatasourceSchema(); } final long pollEndTime = System.nanoTime(); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java index 8b7d885ed64b..3003b99fbe1c 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java @@ -19,9 +19,7 @@ package org.apache.druid.sql.calcite.schema; -import com.google.common.base.Preconditions; import com.google.inject.Binder; -import com.google.inject.Inject; import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.Scopes; @@ -40,7 +38,7 @@ public class DruidCalciteSchemaModule implements Module { private static final String DRUID_SCHEMA_NAME = "druid"; private static final String INFORMATION_SCHEMA_NAME = "INFORMATION_SCHEMA"; - private static final String SEGMENT_METADATA_CACHE_DISABLED = "druid.sql.planner.disableSegmentMetadataCache"; + private static final String SEGMENT_METADATA_CACHE_ENABLED = "druid.sql.planner.segmentMetadataCacheEnabled"; static final String INCOMPLETE_SCHEMA = "INCOMPLETE_SCHEMA"; private final Properties properties; @@ -62,7 +60,7 @@ public void configure(Binder binder) .in(Scopes.SINGLETON); // BrokerSegmentMetadataCache needs to listen to changes for incoming segments - if (!isSegmentMetadataCacheDisabled()) { + if (isSegmentMetadataCacheEnabled()) { LifecycleModule.register(binder, BrokerSegmentMetadataCache.class); } binder.bind(DruidSchema.class).in(Scopes.SINGLETON); @@ -85,8 +83,8 @@ private DruidSchemaCatalog getRootSchema(@Named(INCOMPLETE_SCHEMA) DruidSchemaCa return rootSchema; } - private boolean isSegmentMetadataCacheDisabled() + private boolean isSegmentMetadataCacheEnabled() { - return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_DISABLED)); + return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_ENABLED, "true")); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 73610534493a..38491de49507 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -72,6 +72,7 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.DruidNode; +import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.AuthenticationResult; @@ -110,10 +111,10 @@ public class SystemSchema extends AbstractSchema private static final String TASKS_TABLE = "tasks"; private static final String SUPERVISOR_TABLE = "supervisors"; - private static final Function> - SEGMENT_STATUS_IN_CLUSTER_RA_GENERATOR = segment -> + private static final Function> + SEGMENT_STATUS_IN_CLUSTER_RA_GENERATOR = segmentTableView -> Collections.singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply( - segment.getDataSegment().getDataSource()) + segmentTableView.getSegment().getDataSource()) ); private static final Function> SEGMENT_RA_GENERATOR = @@ -278,41 +279,12 @@ public TableType getJdbcTableType() @Override public Enumerable scan(DataContext root) { - final SegmentTableView segmentTableView = metadataView.getSegmentTableView(); - - final Map availableSegmentMetadata = - segmentTableView.getAvailableSegmentMetadata(); - final Iterator> availableSegmentEntries = - availableSegmentMetadata.entrySet().iterator(); - - // in memory map to store segment data from available segments - final Map partialSegmentDataMap = - Maps.newHashMapWithExpectedSize(segmentTableView.totalSegmentCount()); - for (AvailableSegmentMetadata h : availableSegmentMetadata.values()) { - PartialSegmentData partialSegmentData = - new PartialSegmentData(IS_AVAILABLE_TRUE, h.isRealtime(), h.getNumReplicas(), h.getNumRows()); - partialSegmentDataMap.put(h.getSegment().getId(), partialSegmentData); - } - - // Get published segments from metadata segment cache (if enabled in SQL planner config), else directly from - // Coordinator. - final Iterator metadataStoreSegments = segmentTableView.getPublishedSegments(); - - final Set segmentsAlreadySeen = Sets.newHashSetWithExpectedSize(segmentTableView.totalSegmentCount()); + final Iterator segmentTableView = metadataView.getSegmentTableView(); - final FluentIterable publishedSegments = FluentIterable - .from(() -> getAuthorizedPublishedSegments(metadataStoreSegments, root)) + final FluentIterable allSegments = FluentIterable + .from(() -> getAuthorizedPublishedSegments(segmentTableView, root)) .transform(val -> { - final DataSegment segment = val.getDataSegment(); - segmentsAlreadySeen.add(segment.getId()); - final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(segment.getId()); - long numReplicas = 0L, numRows = 0L, isRealtime = 0L, isAvailable = 0L; - if (partialSegmentData != null) { - numReplicas = partialSegmentData.getNumReplicas(); - numRows = partialSegmentData.getNumRows(); - isAvailable = partialSegmentData.isAvailable(); - isRealtime = partialSegmentData.isRealtime(); - } + final DataSegment segment = val.getSegment(); try { return new Object[]{ segment.getId(), @@ -322,19 +294,21 @@ public Enumerable scan(DataContext root) segment.getSize(), segment.getVersion(), (long) segment.getShardSpec().getPartitionNum(), - numReplicas, - numRows, + val.getNumReplicas(), + val.getNumRows(), //is_active is true for published segments that are not overshadowed val.isOvershadowed() ? IS_ACTIVE_FALSE : IS_ACTIVE_TRUE, //is_published is true for published segments - IS_PUBLISHED_TRUE, - isAvailable, - isRealtime, + val.isPublished ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE, + val.getIsAvailable(), + val.getIsRealtime(), val.isOvershadowed() ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE, segment.getShardSpec() == null ? null : jsonMapper.writeValueAsString(segment.getShardSpec()), segment.getDimensions() == null ? null : jsonMapper.writeValueAsString(segment.getDimensions()), segment.getMetrics() == null ? null : jsonMapper.writeValueAsString(segment.getMetrics()), - segment.getLastCompactionState() == null ? null : jsonMapper.writeValueAsString(segment.getLastCompactionState()), + segment.getLastCompactionState() == null + ? null + : jsonMapper.writeValueAsString(segment.getLastCompactionState()), // If the value is null, the load rules might have not evaluated yet, and we don't know the replication factor. // This should be automatically updated in the next refesh with Coordinator. val.getReplicationFactor() == null ? REPLICATION_FACTOR_UNKNOWN : (long) val.getReplicationFactor() @@ -345,59 +319,12 @@ public Enumerable scan(DataContext root) } }); - final FluentIterable availableSegments = FluentIterable - .from(() -> getAuthorizedAvailableSegments( - availableSegmentEntries, - root - )) - .transform(val -> { - if (segmentsAlreadySeen.contains(val.getKey())) { - return null; - } - final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(val.getKey()); - final long numReplicas = partialSegmentData == null ? 0L : partialSegmentData.getNumReplicas(); - try { - return new Object[]{ - val.getKey(), - val.getKey().getDataSource(), - val.getKey().getInterval().getStart().toString(), - val.getKey().getInterval().getEnd().toString(), - val.getValue().getSegment().getSize(), - val.getKey().getVersion(), - (long) val.getValue().getSegment().getShardSpec().getPartitionNum(), - numReplicas, - val.getValue().getNumRows(), - // is_active is true for unpublished segments iff they are realtime - val.getValue().isRealtime() /* is_active */, - // is_published is false for unpublished segments - IS_PUBLISHED_FALSE, - // is_available is assumed to be always true for segments announced by historicals or realtime tasks - IS_AVAILABLE_TRUE, - val.getValue().isRealtime(), - IS_OVERSHADOWED_FALSE, - // there is an assumption here that unpublished segments are never overshadowed - val.getValue().getSegment().getShardSpec() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getShardSpec()), - val.getValue().getSegment().getDimensions() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getDimensions()), - val.getValue().getSegment().getMetrics() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getMetrics()), - null, // unpublished segments from realtime tasks will not be compacted yet - REPLICATION_FACTOR_UNKNOWN // If the segment is unpublished, we won't have this information yet. - }; - } - catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }); - - final Iterable allSegments = Iterables.unmodifiableIterable( - Iterables.concat(publishedSegments, availableSegments) - ); return Linq4j.asEnumerable(allSegments).where(Objects::nonNull); - } - private Iterator getAuthorizedPublishedSegments( - Iterator it, + private Iterator getAuthorizedPublishedSegments( + Iterator it, DataContext root ) { @@ -406,7 +333,7 @@ private Iterator getAuthorizedPublishedSegments( "authenticationResult in dataContext" ); - final Iterable authorizedSegments = AuthorizationUtils + final Iterable authorizedSegments = AuthorizationUtils .filterAuthorizedResources( authenticationResult, () -> it, @@ -443,6 +370,79 @@ private Iterator> getAuthorizedAvaila } protected static class SegmentTableView + { + private final DataSegment segment; + private final long isAvailable; + private final long isRealtime; + private final long numReplicas; + private final long numRows; + private final Integer replicationFactor; + private final boolean isOvershadowed; + private final boolean isPublished; + + public SegmentTableView( + DataSegment segment, + long isAvailable, + long isRealtime, + long numReplicas, + long numRows, + Integer replicationFactor, + boolean isOvershadowed, + boolean isPublished + ) + { + this.segment = segment; + this.isAvailable = isAvailable; + this.isRealtime = isRealtime; + this.numReplicas = numReplicas; + this.numRows = numRows; + this.replicationFactor = replicationFactor; + this.isOvershadowed = isOvershadowed; + this.isPublished = isPublished; + } + + public DataSegment getSegment() + { + return segment; + } + + public long getIsAvailable() + { + return isAvailable; + } + + public long getIsRealtime() + { + return isRealtime; + } + + public long getNumReplicas() + { + return numReplicas; + } + + public long getNumRows() + { + return numRows; + } + + public Integer getReplicationFactor() + { + return replicationFactor; + } + + public boolean isOvershadowed() + { + return isOvershadowed; + } + + public boolean isPublished() + { + return isPublished; + } + } + + protected static class SegmentTableView1 { // pass the stitched private final Map availableSegmentMetadata; @@ -450,7 +450,7 @@ protected static class SegmentTableView private final int totalSegmentsCount; - public SegmentTableView( + public SegmentTableView1( Map availableSegmentMetadata, ImmutableSortedSet publishedSegments ) @@ -489,7 +489,6 @@ public PartialSegmentData( final long numReplicas, final long numRows ) - { this.isAvailable = isAvailable; this.isRealtime = isRealtime; From 105bda960390fdbae9592f00893ecd145dd630f3 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 6 Sep 2023 14:43:35 +0530 Subject: [PATCH 06/36] Move schema querying logic to BrokerSegmentMetadataCache --- .../timeline/SegmentStatusInCluster.java | 30 ++-- .../metadata/SegmentMetadataCache.java | 16 +- .../druid/server/http/MetadataResource.java | 88 +++++---- .../org/apache/druid/cli/CliCoordinator.java | 1 + .../schema/BrokerSegmentMetadataCache.java | 84 ++++++++- .../BrokerSegmentMetadataCacheConfig.java | 7 - .../schema/BrokerSegmentMetadataView.java | 170 +++++------------- .../druid/sql/calcite/schema/DruidSchema.java | 13 +- .../sql/calcite/schema/SystemSchema.java | 27 +-- .../schema/SegmentMetadataCacheTest.java | 4 +- 10 files changed, 216 insertions(+), 224 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java index 263f1619c6f3..bccfa764726b 100644 --- a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java +++ b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java @@ -58,21 +58,21 @@ public class SegmentStatusInCluster implements Comparable dataSources, @QueryParam("includeOvershadowedStatus") final @Nullable String includeOvershadowedStatus, - @QueryParam("includeOvershadowedStatus") final @Nullable String includeRealtime + @QueryParam("includeRealtimeSegments") final @Nullable String includeRealtimeSegments ) { if (includeOvershadowedStatus != null) { - return getAllUsedSegmentsWithAdditionalDetails(req, dataSources); + return getAllUsedSegmentsWithAdditionalDetails(req, dataSources, includeRealtimeSegments); } Collection dataSourcesWithUsedSegments = @@ -180,7 +181,8 @@ public Response getAllUsedSegments( private Response getAllUsedSegmentsWithAdditionalDetails( HttpServletRequest req, - @Nullable Set dataSources + @Nullable Set dataSources, + String includeRealtimeSegments ) { DataSourcesSnapshot dataSourcesSnapshot = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments(); @@ -200,44 +202,61 @@ private Response getAllUsedSegmentsWithAdditionalDetails( .map(segment -> { // The replication factor for unloaded segments is 0 as they will be unloaded soon boolean isOvershadowed = overshadowedSegments.contains(segment); - AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata( - segment.getDataSource(), - segment.getId() - ); + Long numRows = null; + if (null != segmentMetadataCache) { + AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata( + segment.getDataSource(), + segment.getId() + ); + if (null != availableSegmentMetadata) { + numRows = availableSegmentMetadata.getNumRows(); + } + } Integer replicationFactor = isOvershadowed ? (Integer) 0 : coordinator.getReplicationFactor(segment.getId()); - Long numRows = (null != availableSegmentMetadata) ? availableSegmentMetadata.getNumRows() : null; segmentAlreadySeen.add(segment.getId()); - return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, 20L, true, 0L); - }).peek(v -> log.info("peeking into first stream segmentseen [%s], id [%s] isPublished [%s]", segmentAlreadySeen, v.getDataSegment().getId(), v.isPublished())); + return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, 20L, true); + }).peek(v -> log.info("peeking into first stream segmentseen [%s], id [%s]", segmentAlreadySeen, v.getDataSegment().getId())); - log.info("printing the content in smc cache %s", segmentMetadataCache.getSegmentMetadataSnapshot().values()); + Stream finalSegments = segmentStatus; + + if (null != includeRealtimeSegments && segmentMetadataCache != null) { + log.info("printing the content in smc cache %s", segmentMetadataCache.getSegmentMetadataSnapshot().values()); + final Stream realtimeSegmentStatus = segmentMetadataCache + .getSegmentMetadataSnapshot() + .values() + .stream() + .peek(v -> log.info( + "peeking into second stream (first) segmentseen [%s], id [%s]", + segmentAlreadySeen, + v.getSegment().getId() + )) + .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment() + .getId())) + .map(availableSegmentMetadata -> { + return new SegmentStatusInCluster( + availableSegmentMetadata.getSegment(), + false, + (int) availableSegmentMetadata.getNumReplicas(), + /**availableSegmentMetadata.getNumRows(), **/30L, + availableSegmentMetadata.isRealtime() != 0 + ); + }) + .peek(v -> log.info("peeking into second stream (second) segmentseen [%s], id [%s]", + segmentAlreadySeen, + v.getDataSegment().getId() + )); + + finalSegments = Stream.concat(segmentStatus, realtimeSegmentStatus) + .peek(v -> log.info("combined stream element %s", v)); + } - final Stream realtimeSegmentStatus = segmentMetadataCache - .getSegmentMetadataSnapshot() - .values() - .stream() - .peek(v -> log.info("peeking into second stream (first) segmentseen [%s], id [%s]", segmentAlreadySeen, v.getSegment().getId())) - .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment().getId())) - .map(availableSegmentMetadata -> { - return new SegmentStatusInCluster( - availableSegmentMetadata.getSegment(), - false, - (int) availableSegmentMetadata.getNumReplicas(), - /**availableSegmentMetadata.getNumRows(), **/ 30L, - false, - availableSegmentMetadata.isRealtime() - ); - }).peek(v -> log.info("peeking into second stream (second) segmentseen [%s], id [%s]", segmentAlreadySeen, v.getDataSegment().getId()));; - - - Stream combined = Stream.concat(segmentStatus, realtimeSegmentStatus).peek(v -> log.info("combined stream element %s", v)); final Function> raGenerator = segment -> Collections .singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getDataSegment().getDataSource())); final Iterable authorizedSegments = AuthorizationUtils.filterAuthorizedResources( req, - combined::iterator, + finalSegments::iterator, raGenerator, authorizerMapper ); @@ -347,6 +366,9 @@ public Response getDatasourceSchema( List datasources ) { + if (null == segmentMetadataCache) { + return Response.status(Response.Status.NOT_FOUND).build(); + } Map datasourceSchemaMap = segmentMetadataCache.getDatasourceSchemaMap(); datasourceSchemaMap.keySet().retainAll(datasources); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 85c312f2b04e..6c16684a2e93 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -233,6 +233,7 @@ public void configure(Binder binder) } else { binder.bind(CoordinatorInventoryView.class).to(CoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, CoordinatorServerView.class); + binder.bind(SegmentMetadataCache.class).toProvider(Providers.of(null)); } binder.bind(SegmentsMetadataManager.class) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 09f252cc3e20..a7f47d6c5fde 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -1,9 +1,11 @@ package org.apache.druid.sql.calcite.schema; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; import com.google.inject.Inject; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; +import org.apache.druid.client.coordinator.CoordinatorClient; +import org.apache.druid.common.guava.FutureUtils; import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; @@ -13,7 +15,11 @@ import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.timeline.SegmentId; +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -25,20 +31,23 @@ @ManageLifecycle public class BrokerSegmentMetadataCache extends SegmentMetadataCache { - private static final EmittingLogger log = new EmittingLogger(SegmentMetadataCache.class); + private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataCache.class); private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final CoordinatorClient coordinatorClient; + @Inject public BrokerSegmentMetadataCache( - QueryLifecycleFactory queryLifecycleFactory, - TimelineServerView serverView, - SegmentMetadataCacheConfig config, - Escalator escalator, - InternalQueryConfig internalQueryConfig, - ServiceEmitter emitter, - PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder + final QueryLifecycleFactory queryLifecycleFactory, + final TimelineServerView serverView, + final SegmentMetadataCacheConfig config, + final Escalator escalator, + final InternalQueryConfig internalQueryConfig, + final ServiceEmitter emitter, + final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder, + final CoordinatorClient coordinatorClient ) { super( @@ -50,6 +59,63 @@ public BrokerSegmentMetadataCache( emitter ); this.physicalDatasourceMetadataBuilder = physicalDatasourceMetadataBuilder; + this.coordinatorClient = coordinatorClient; + } + + @Override + public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException + { + Set dataSourcesToQuery = new HashSet<>(); + + segmentsToRefresh.forEach(segment -> dataSourcesToQuery.add(segment.getDataSource())); + + Map polledDataSourceSchema = new HashMap<>(); + + // Fetch dataSource schema from the Coordinator + try { + FutureUtils.getUnchecked(coordinatorClient.fetchDatasourceSchema(dataSourcesToQuery), true) + .forEach(item -> polledDataSourceSchema.put( + item.getDatasource(), + physicalDatasourceMetadataBuilder.build( + item.getDatasource(), + item.getRowSignature() + ) + )); + } catch (Exception e) { + log.error("Exception querying coordinator for schema"); + } + + log.info("Queried ds schema are [%s]", polledDataSourceSchema); + + tables.putAll(polledDataSourceSchema); + + // Remove segments of the dataSource from refresh list for which we received schema from the Coordinator. + segmentsToRefresh.forEach(segment -> { + if (polledDataSourceSchema.containsKey(segment.getDataSource())) { + segmentsToRefresh.remove(segment); + } + }); + + // Refresh the segments. + final Set refreshed = refreshSegments(segmentsToRefresh); + + synchronized (getLock()) { + // Add missing segments back to the refresh list. + getSegmentsNeedingRefresh().addAll(Sets.difference(segmentsToRefresh, refreshed)); + + // Compute the list of dataSources to rebuild tables for. + dataSourcesToRebuild.addAll(getDataSourcesNeedingRebuild()); + refreshed.forEach(segment -> dataSourcesToRebuild.add(segment.getDataSource())); + + // Remove those dataSource for which we received schema from the Coordinator. + dataSourcesToRebuild.removeAll(polledDataSourceSchema.keySet()); + getDataSourcesNeedingRebuild().clear(); + } + + // Rebuild the dataSources. + for (String dataSource : dataSourcesToRebuild) { + rebuildDatasource(dataSource); + } } @Override diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 9219706940fe..7f4041fb0b03 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -11,9 +11,6 @@ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig @JsonProperty private long metadataSegmentPollPeriod = 60000; - @JsonProperty - private boolean segmentMetadataCacheEnabled = false; - public boolean isMetadataSegmentCacheEnable() { return metadataSegmentCacheEnable; @@ -24,8 +21,4 @@ public long getMetadataSegmentPollPeriod() return metadataSegmentPollPeriod; } - public boolean isSegmentMetadataCacheEnabled() - { - return segmentMetadataCacheEnabled; - } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index bb1d4f4ec6ff..9eeafcd4f8c4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -32,9 +32,7 @@ import org.apache.druid.client.DataSegmentInterner; import org.apache.druid.client.JsonParserIterator; import org.apache.druid.client.coordinator.Coordinator; -import org.apache.druid.client.coordinator.CoordinatorClient; import org.apache.druid.client.selector.ServerSelector; -import org.apache.druid.common.guava.FutureUtils; import org.apache.druid.concurrent.LifecycleLock; import org.apache.druid.discovery.DruidLeaderClient; import org.apache.druid.guice.ManageLifecycle; @@ -45,16 +43,13 @@ import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; -import org.apache.druid.segment.metadata.DatasourceSchema; import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable.SegmentTableView; -import org.apache.druid.sql.calcite.table.DatasourceTable; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -76,20 +71,13 @@ public class BrokerSegmentMetadataView { private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataView.class); - private final CoordinatorClient coordinatorClient; private final BrokerSegmentWatcherConfig segmentWatcherConfig; private final DruidLeaderClient druidLeaderClient; private final ObjectMapper objectMapper; - private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; private final boolean isMetadataSegmentCacheEnabled; - private final boolean useSegmentMetadataCache; - - private final boolean includeRealtimeSegments; - - private final boolean pollDsSchema; /** * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and @@ -102,8 +90,6 @@ public class BrokerSegmentMetadataView @MonotonicNonNull private volatile ImmutableSortedSet segmentMetadata = null; - private volatile Map datasourceSchemaMap = null; - private final BrokerServerView brokerServerView; private final BrokerSegmentMetadataCache segmentMetadataCache; @@ -123,10 +109,8 @@ public class BrokerSegmentMetadataView public BrokerSegmentMetadataView( final @Coordinator DruidLeaderClient druidLeaderClient, final ObjectMapper objectMapper, - final CoordinatorClient coordinatorClient, final BrokerSegmentWatcherConfig segmentWatcherConfig, final BrokerSegmentMetadataCacheConfig config, - final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder, final BrokerServerView brokerServerView, final BrokerSegmentMetadataCache segmentMetadataCache ) @@ -134,20 +118,15 @@ public BrokerSegmentMetadataView( Preconditions.checkNotNull(config, "BrokerSegmentMetadataCacheConfig"); this.druidLeaderClient = druidLeaderClient; this.objectMapper = objectMapper; - this.coordinatorClient = coordinatorClient; this.segmentWatcherConfig = segmentWatcherConfig; this.isMetadataSegmentCacheEnabled = config.isMetadataSegmentCacheEnable(); - this.useSegmentMetadataCache = config.isSegmentMetadataCacheEnabled(); - this.includeRealtimeSegments = !useSegmentMetadataCache; - this.pollDsSchema = !useSegmentMetadataCache; this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); this.scheduledExec = Execs.scheduledSingleThreaded("SegmentMetadataView-Cache--%d"); this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() .expireAfterAccess(10, TimeUnit.MINUTES) .build(); - this.physicalDatasourceMetadataBuilder = physicalDatasourceMetadataBuilder; this.brokerServerView = brokerServerView; this.segmentMetadataCache = segmentMetadataCache; } @@ -159,12 +138,12 @@ public void start() throw new ISE("can't start."); } try { - if (isMetadataSegmentCacheEnabled || pollDsSchema) { + if (isMetadataSegmentCacheEnabled) { scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); } lifecycleLock.started(); - log.info("MetadataSegmentView Started. Configs isMetadataSegmentCacheEnabled [%s], useSegmentMetadataCache [%s]", - isMetadataSegmentCacheEnabled, useSegmentMetadataCache); + log.info("MetadataSegmentView Started. Configs isMetadataSegmentCacheEnabled [%s]", + isMetadataSegmentCacheEnabled); } finally { lifecycleLock.exitStart(); @@ -178,57 +157,47 @@ public void stop() throw new ISE("can't stop."); } log.info("MetadataSegmentView is stopping."); - if (isMetadataSegmentCacheEnabled || pollDsSchema) { + if (isMetadataSegmentCacheEnabled) { scheduledExec.shutdown(); } log.info("MetadataSegmentView Stopped."); } - protected DatasourceTable.PhysicalDatasourceMetadata getDatasource(String name) - { - if (useSegmentMetadataCache) { - return segmentMetadataCache.getPhysicalDatasourceMetadata(name); - } - return datasourceSchemaMap.get(name); - } - - protected Set getDatasourceNames() + protected Iterator getSegmentTableView() { - if (useSegmentMetadataCache) { - return segmentMetadataCache.getDatasourceNames(); - } - return datasourceSchemaMap.keySet(); - } - - protected Iterator getSegmentTableView() { - if (useSegmentMetadataCache) { - return getSegmentTableViewFromCoordinatorAndSmc(); - } - return getSegmentTableViewFromCoordinator(); - } - - private Iterator getSegmentTableViewFromCoordinatorAndSmc() - { - final ImmutableSortedSet publishedSegments = getSegmentMetadata(); + final ImmutableSortedSet segments = getSegmentMetadata(); final Map availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); - - final List segmentsTableView = new ArrayList<>(); + final Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); + final List segmentsTableViews = new ArrayList<>(); Set seenSegments = new HashSet<>(); - for (SegmentStatusInCluster segmentStatusInCluster : publishedSegments) + + for (SegmentStatusInCluster segmentStatusInCluster : segments) { DataSegment segment = segmentStatusInCluster.getDataSegment(); SegmentId segmentId = segment.getId(); AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataMap.get(segmentId); - long numReplicas = 0L, numRows = 0L, isRealtime = 0L, isAvailable = 0L; + long numReplicas = 0L, numRows = 0L, isAvailable = 0L; if (availableSegmentMetadata != null) { numReplicas = availableSegmentMetadata.getNumReplicas(); numRows = availableSegmentMetadata.getNumRows(); isAvailable = 1L; - isRealtime = availableSegmentMetadata.isRealtime(); + } else if (brokerSegmentMetadata.containsKey(segmentId)) { + ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); + numReplicas = serverSelector.getAllServers().size(); + isAvailable = 1L; + } + + // Prefer numRows & realtime status info returned from Coordinator. + if (null != segmentStatusInCluster.getNumRows()) + { + numRows = segmentStatusInCluster.getNumRows(); } + long isRealtime = Boolean.TRUE.equals(segmentStatusInCluster.isRealtime()) ? 1 : 0; + + SegmentTableView segmentTableView = new SegmentTableView( segment, isAvailable, @@ -236,11 +205,10 @@ private Iterator getSegmentTableViewFromCoordinatorAndSmc() numReplicas, numRows, segmentStatusInCluster.getReplicationFactor(), - segmentStatusInCluster.isOvershadowed(), - true + segmentStatusInCluster.isOvershadowed() ); seenSegments.add(segmentId); - segmentsTableView.add((segmentTableView)); + segmentsTableViews.add((segmentTableView)); } for (Map.Entry availableSegmentMetadataEntry : availableSegmentMetadataMap.entrySet()) @@ -256,70 +224,16 @@ private Iterator getSegmentTableViewFromCoordinatorAndSmc() availableSegmentMetadata.getNumReplicas(), availableSegmentMetadata.getNumRows(), null, - false, false ); - segmentsTableView.add(segmentTableView); - } - - return segmentsTableView.iterator(); - } - - private Iterator getSegmentTableViewFromCoordinator() - { - final ImmutableSortedSet allSegments = getSegmentMetadata(); - final Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); - - final List segmentsTableView = new ArrayList<>(); - - for (SegmentStatusInCluster segmentStatusInCluster : allSegments) { - SegmentId segmentId = segmentStatusInCluster.getDataSegment().getId(); - ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); - long numReplicas = 0L, isAvailable = 0L, numRows = 0L; - if (null != serverSelector) { - numReplicas = serverSelector.getAllServers().size(); - isAvailable = 1L; - } - if (null != segmentStatusInCluster.getNumRows()) - { - numRows = segmentStatusInCluster.getNumRows(); - } - - SegmentTableView segmentTableView = new SegmentTableView( - segmentStatusInCluster.getDataSegment(), - isAvailable, - segmentStatusInCluster.isRealtime(), - numReplicas, - numRows, - segmentStatusInCluster.getReplicationFactor(), - segmentStatusInCluster.isOvershadowed(), - segmentStatusInCluster.isPublished() - ); - segmentsTableView.add(segmentTableView); + segmentsTableViews.add(segmentTableView); } - return segmentsTableView.iterator(); - } - - private void pollDatasourceSchema() - { - log.info("Polling datasource schema from coordinator."); - - Set watchedDatasources = segmentWatcherConfig.getWatchedDataSources(); - - Map physicalDatasourceMetadataMap = new HashMap<>(); - - List datasourceSchemas = FutureUtils.getUnchecked( - coordinatorClient.fetchDatasourceSchema(watchedDatasources), true); - - for (DatasourceSchema datasourceSchema : datasourceSchemas) { - physicalDatasourceMetadataMap.put( - datasourceSchema.getDatasource(), - physicalDatasourceMetadataBuilder.build(datasourceSchema.getDatasource(), datasourceSchema.getRowSignature()) - ); + log.info("Built the segment table view"); + for (SegmentTableView segmentTableView : segmentsTableViews) { + log.info("SegmentTableView is [%s]", segmentTableView); } - - this.datasourceSchemaMap = physicalDatasourceMetadataMap; + return segmentsTableViews.iterator(); } private void pollSegmentMetadata() @@ -346,8 +260,10 @@ private ImmutableSortedSet fetchSegmentMetadata() querySegmentMetadata(segmentWatcherConfig.getWatchedDataSources()); final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); + log.info("Polled segments from coordinator"); while (metadataSegments.hasNext()) { final SegmentStatusInCluster segment = metadataSegments.next(); + log.info("This is the polled segmentStatusInCluster %s", segment); final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); Integer replicationFactor = segment.getReplicationFactor(); if (replicationFactor == null) { @@ -360,23 +276,22 @@ private ImmutableSortedSet fetchSegmentMetadata() segment.isOvershadowed(), replicationFactor, segment.getNumRows(), - true, segment.isRealtime() ); + log.info("SegmentStatusInCluster %s", segmentStatusInCluster); builder.add(segmentStatusInCluster); } + return builder.build(); } - // Note that coordinator must be up to get segments private JsonParserIterator querySegmentMetadata( Set watchedDataSources ) { final StringBuilder queryBuilder = new StringBuilder("/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"); - if (includeRealtimeSegments) { - queryBuilder.append("&includeRealtimeSegments"); - } + + //queryBuilder.append("&includeRealtimeSegments"); if (watchedDataSources != null && !watchedDataSources.isEmpty()) { log.debug( "filtering datasources in published segments based on broker's watchedDataSources[%s]", watchedDataSources); @@ -386,8 +301,10 @@ private JsonParserIterator querySegmentMetadata( queryBuilder.setLength(queryBuilder.length() - 1); } + String query = queryBuilder.toString(); + log.info("query is %s", query); return SystemSchema.getThingsFromLeaderNode( - queryBuilder.toString(), + query, new TypeReference() { }, @@ -404,12 +321,7 @@ public void run() long delayMS = pollPeriodInMS; try { final long pollStartTime = System.nanoTime(); - if (isMetadataSegmentCacheEnabled) { - pollSegmentMetadata(); - } - if (pollDsSchema) { - pollDatasourceSchema(); - } + pollSegmentMetadata(); final long pollEndTime = System.nanoTime(); final long pollTimeNS = pollEndTime - pollStartTime; final long pollTimeMS = TimeUnit.NANOSECONDS.toMillis(pollTimeNS); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java index 28a6ab252516..7f4f626e6ec6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java @@ -20,6 +20,7 @@ package org.apache.druid.sql.calcite.schema; import org.apache.calcite.schema.Table; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.sql.calcite.table.DatasourceTable; import javax.inject.Inject; @@ -28,16 +29,16 @@ public class DruidSchema extends AbstractTableSchema { - private final BrokerSegmentMetadataView brokerSegmentMetadataView; + private final BrokerSegmentMetadataCache segmentMetadataCache; private final DruidSchemaManager druidSchemaManager; @Inject public DruidSchema( - final BrokerSegmentMetadataView brokerSegmentMetadataView, + final BrokerSegmentMetadataCache segmentMetadataCache, final DruidSchemaManager druidSchemaManager ) { - this.brokerSegmentMetadataView = brokerSegmentMetadataView; + this.segmentMetadataCache = segmentMetadataCache; if (druidSchemaManager != null && !(druidSchemaManager instanceof NoopDruidSchemaManager)) { this.druidSchemaManager = druidSchemaManager; } else { @@ -51,9 +52,9 @@ public Table getTable(String name) if (druidSchemaManager != null) { return druidSchemaManager.getTable(name); } else { - DatasourceTable.PhysicalDatasourceMetadata dsMetadata = brokerSegmentMetadataView.getDatasource(name); + DatasourceTable.PhysicalDatasourceMetadata dsMetadata = segmentMetadataCache.getPhysicalDatasourceMetadata(name); return dsMetadata == null ? null : new DatasourceTable(dsMetadata); -} + } } @Override @@ -62,7 +63,7 @@ public Set getTableNames() if (druidSchemaManager != null) { return druidSchemaManager.getTableNames(); } else { - return brokerSegmentMetadataView.getDatasourceNames(); + return segmentMetadataCache.getDatasourceNames(); } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 38491de49507..28e322a3ddca 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -133,7 +133,6 @@ public class SystemSchema extends AbstractSchema private static final long IS_ACTIVE_TRUE = 1L; private static final long IS_PUBLISHED_FALSE = 0L; private static final long IS_PUBLISHED_TRUE = 1L; - private static final long IS_AVAILABLE_TRUE = 1L; private static final long IS_OVERSHADOWED_FALSE = 0L; private static final long IS_OVERSHADOWED_TRUE = 1L; @@ -214,7 +213,6 @@ public class SystemSchema extends AbstractSchema @Inject public SystemSchema( - final DruidSchema druidSchema, final BrokerSegmentMetadataView metadataView, final TimelineServerView serverView, final FilteredServerInventoryView serverInventoryView, @@ -227,7 +225,7 @@ public SystemSchema( { Preconditions.checkNotNull(serverView, "serverView"); this.tableMap = ImmutableMap.of( - SEGMENTS_TABLE, new SegmentsTable(druidSchema, metadataView, jsonMapper, authorizerMapper), + SEGMENTS_TABLE, new SegmentsTable(metadataView, jsonMapper, authorizerMapper), SERVERS_TABLE, new ServersTable(druidNodeDiscoveryProvider, serverInventoryView, authorizerMapper, overlordClient, coordinatorDruidLeaderClient), SERVER_SEGMENTS_TABLE, new ServerSegmentsTable(serverView, authorizerMapper), TASKS_TABLE, new TasksTable(overlordClient, authorizerMapper), @@ -246,19 +244,16 @@ public Map getTableMap() */ static class SegmentsTable extends AbstractTable implements ScannableTable { - private final DruidSchema druidSchema; private final ObjectMapper jsonMapper; private final AuthorizerMapper authorizerMapper; private final BrokerSegmentMetadataView metadataView; public SegmentsTable( - DruidSchema druidSchemna, BrokerSegmentMetadataView metadataView, ObjectMapper jsonMapper, AuthorizerMapper authorizerMapper ) { - this.druidSchema = druidSchemna; this.metadataView = metadataView; this.jsonMapper = jsonMapper; this.authorizerMapper = authorizerMapper; @@ -299,7 +294,7 @@ public Enumerable scan(DataContext root) //is_active is true for published segments that are not overshadowed val.isOvershadowed() ? IS_ACTIVE_FALSE : IS_ACTIVE_TRUE, //is_published is true for published segments - val.isPublished ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE, + (val.getIsRealtime() == 0) ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE, val.getIsAvailable(), val.getIsRealtime(), val.isOvershadowed() ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE, @@ -378,7 +373,6 @@ protected static class SegmentTableView private final long numRows; private final Integer replicationFactor; private final boolean isOvershadowed; - private final boolean isPublished; public SegmentTableView( DataSegment segment, @@ -387,8 +381,7 @@ public SegmentTableView( long numReplicas, long numRows, Integer replicationFactor, - boolean isOvershadowed, - boolean isPublished + boolean isOvershadowed ) { this.segment = segment; @@ -398,7 +391,6 @@ public SegmentTableView( this.numRows = numRows; this.replicationFactor = replicationFactor; this.isOvershadowed = isOvershadowed; - this.isPublished = isPublished; } public DataSegment getSegment() @@ -436,9 +428,18 @@ public boolean isOvershadowed() return isOvershadowed; } - public boolean isPublished() + @Override + public String toString() { - return isPublished; + return "SegmentTableView{" + + "segmentId=" + segment.getId() + + ", isAvailable=" + isAvailable + + ", isRealtime=" + isRealtime + + ", numReplicas=" + numReplicas + + ", numRows=" + numRows + + ", replicationFactor=" + replicationFactor + + ", isOvershadowed=" + isOvershadowed + + '}'; } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java index ae5f14aab8c6..2cf08d7d6b71 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java @@ -344,7 +344,7 @@ public SegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedE ) { @Override - void markDataSourceAsNeedRebuild(String datasource) + public void markDataSourceAsNeedRebuild(String datasource) { super.markDataSourceAsNeedRebuild(datasource); markDataSourceLatch.countDown(); @@ -352,7 +352,7 @@ void markDataSourceAsNeedRebuild(String datasource) @Override @VisibleForTesting - void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException + public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException { super.refresh(segmentsToRefresh, dataSourcesToRebuild); refreshLatch.countDown(); From 3aad095a8a547522a139e7fdd7babd40450dabf3 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 6 Sep 2023 18:45:02 +0530 Subject: [PATCH 07/36] Fix dataSource schema on coordinator and minor renaming --- .../client/coordinator/CoordinatorClient.java | 6 ++-- .../coordinator/CoordinatorClientImpl.java | 14 +++++----- ...ourceSchema.java => DataSourceSchema.java} | 4 +-- .../metadata/SegmentMetadataCache.java | 16 +++++------ .../metadata/SegmentMetadataCacheConfig.java | 8 +++++- .../druid/server/http/MetadataResource.java | 28 ++++++++++++------- .../org/apache/druid/cli/CliCoordinator.java | 2 +- .../schema/BrokerSegmentMetadataCache.java | 6 ++-- .../BrokerSegmentMetadataCacheConfig.java | 8 ++++++ 9 files changed, 56 insertions(+), 36 deletions(-) rename server/src/main/java/org/apache/druid/segment/metadata/{DatasourceSchema.java => DataSourceSchema.java} (92%) diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index a56363afee83..ad72159bde02 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -20,14 +20,12 @@ package org.apache.druid.client.coordinator; import com.google.common.util.concurrent.ListenableFuture; -import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentStatusInCluster; import org.joda.time.Interval; -import java.util.Iterator; import java.util.List; import java.util.Set; @@ -48,7 +46,7 @@ public interface CoordinatorClient */ ListenableFuture> fetchUsedSegments(String dataSource, List intervals); - ListenableFuture> fetchDatasourceSchema(Set datasources); + ListenableFuture> fetchDataSourceSchema(Set datasources); /** * Returns a new instance backed by a ServiceClient which follows the provided retryPolicy diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index 2a9e1891f086..24e6968ad6ab 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -30,7 +30,7 @@ import org.apache.druid.rpc.RequestBuilder; import org.apache.druid.rpc.ServiceClient; import org.apache.druid.rpc.ServiceRetryPolicy; -import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.timeline.DataSegment; import org.jboss.netty.handler.codec.http.HttpMethod; import org.joda.time.Interval; @@ -110,20 +110,20 @@ public ListenableFuture> fetchUsedSegments(String dataSource, } @Override - public ListenableFuture> fetchDatasourceSchema(Set datasources) + public ListenableFuture> fetchDataSourceSchema(Set dataSources) { - final String path = "/druid/coordinator/v1/metadata/datasourceSchema"; - if (null == datasources) + final String path = "/druid/coordinator/v1/metadata/dataSourceSchema"; + if (null == dataSources) { - datasources = new HashSet<>(); + dataSources = new HashSet<>(); } return FutureUtils.transform( client.asyncRequest( new RequestBuilder(HttpMethod.POST, path) - .jsonContent(jsonMapper, datasources), + .jsonContent(jsonMapper, dataSources), new BytesFullResponseHandler() ), - holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), new TypeReference>() {}) + holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), new TypeReference>() {}) ); } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java similarity index 92% rename from server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java rename to server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java index 9fb13fe5f0e5..15e87cf5d072 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DatasourceSchema.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java @@ -4,14 +4,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.segment.column.RowSignature; -public class DatasourceSchema +public class DataSourceSchema { // dsinfo private final String datasource; private final RowSignature rowSignature; @JsonCreator - public DatasourceSchema( + public DataSourceSchema( @JsonProperty("datasource") String datasource, @JsonProperty("rowSignature") RowSignature rowSignature) { diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index da32a86cdc2a..3c20b4bcd779 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -124,7 +124,7 @@ public class SegmentMetadataCache * Map of DataSource -> DruidTable. * This map can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final ConcurrentMap tables = new ConcurrentHashMap<>(); /** * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. @@ -181,7 +181,7 @@ public class SegmentMetadataCache * Currently, there are 2 threads that can access these variables. * * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. - * - {@link #cacheExec} periodically refreshes segment metadata and {@link DatasourceSchema} if necessary + * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceSchema} if necessary * based on the information collected via timeline callbacks. */ private final Object lock = new Object(); @@ -429,13 +429,13 @@ public void refresh(final Set segmentsToRefresh, final Set da public void rebuildDatasource(String dataSource) { - final DatasourceSchema druidTable = buildDruidTable(dataSource); + final DataSourceSchema druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } - final DatasourceSchema oldTable = tables.put(dataSource, druidTable); + final DataSourceSchema oldTable = tables.put(dataSource, druidTable); if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { @@ -455,12 +455,12 @@ public void awaitInitialization() throws InterruptedException initialized.await(); } - public DatasourceSchema getDatasource(String name) + public DataSourceSchema getDatasource(String name) { return tables.get(name); } - public Map getDatasourceSchemaMap() + public Map getDataSourceSchemaMap() { return ImmutableMap.copyOf(tables); } @@ -819,7 +819,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin @VisibleForTesting @Nullable - public DatasourceSchema buildDruidTable(final String dataSource) + public DataSourceSchema buildDruidTable(final String dataSource) { ConcurrentSkipListMap segmentsMap = segmentMetadataInfo.get(dataSource); @@ -847,7 +847,7 @@ public DatasourceSchema buildDruidTable(final String dataSource) final RowSignature.Builder builder = RowSignature.builder(); columnTypes.forEach(builder::add); - return new DatasourceSchema(dataSource, builder.build()); + return new DataSourceSchema(dataSource, builder.build()); } public Map getSegmentMetadataSnapshot() diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index c5d22c087794..80aba42ecddc 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -6,9 +6,10 @@ public class SegmentMetadataCacheConfig { @JsonProperty - private boolean awaitInitializationOnStart = true; + private boolean awaitInitializationOnStart = false; @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); + private boolean enabled = false; @JsonProperty private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); @@ -41,4 +42,9 @@ public Period getMetadataRefreshPeriod() { return metadataRefreshPeriod; } + + public boolean isEnabled() + { + return enabled; + } } diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 69dd113e180b..5b70be95b5b4 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -29,11 +29,10 @@ import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.Segments; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; -import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.segment.metadata.SegmentMetadataCache; -import org.apache.druid.metadata.SegmentsMetadataManager; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.JettyUtils; import org.apache.druid.server.coordinator.DruidCoordinator; import org.apache.druid.server.http.security.DatasourceResourceFilter; @@ -44,7 +43,6 @@ import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.joda.time.Interval; -import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import javax.servlet.http.HttpServletRequest; @@ -58,6 +56,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashSet; @@ -361,17 +360,26 @@ public Response getUsedSegment( } @POST - @Path("/datasourceSchema") - public Response getDatasourceSchema( - List datasources + @Path("/dataSourceSchema") + @Produces(MediaType.APPLICATION_JSON) + public Response getDataSourceSchema( + List dataSources ) { if (null == segmentMetadataCache) { return Response.status(Response.Status.NOT_FOUND).build(); } - Map datasourceSchemaMap = segmentMetadataCache.getDatasourceSchemaMap(); - datasourceSchemaMap.keySet().retainAll(datasources); + Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceSchemaMap(); + + List results = new ArrayList<>(); + List dataSourcesToRetain = (null == dataSources) ? new ArrayList<>(dataSourceSchemaMap.keySet()) : dataSources; + + for (Map.Entry entry : dataSourceSchemaMap.entrySet()) { + if (dataSourcesToRetain.contains(entry.getKey())) { + results.add(entry.getValue()); + } + } - return Response.status(Response.Status.OK).entity(datasourceSchemaMap.values()).build(); + return Response.status(Response.Status.OK).entity(results).build(); } } diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 6c16684a2e93..c9b23df64fb2 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -220,6 +220,7 @@ public void configure(Binder binder) "druid.coordinator.balancer.cachingCost", CachingCostBalancerStrategyConfig.class ); + JsonConfigProvider.bind(binder, "druid.coordinator.segmentMetadataCache", SegmentMetadataCacheConfig.class); binder.bind(RedirectFilter.class).in(LazySingleton.class); if (beOverlord) { @@ -532,7 +533,6 @@ public void configure(Binder binder) binder.install(new BrokerProcessingModule()); binder.install(new JoinableFactoryModule()); - JsonConfigProvider.bind(binder, "druid.coordinator.segment", SegmentMetadataCacheConfig.class); JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index a7f47d6c5fde..8f3cd67abe66 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -9,7 +9,7 @@ import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.segment.metadata.DatasourceSchema; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; @@ -73,7 +73,7 @@ public void refresh(final Set segmentsToRefresh, final Set da // Fetch dataSource schema from the Coordinator try { - FutureUtils.getUnchecked(coordinatorClient.fetchDatasourceSchema(dataSourcesToQuery), true) + FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceSchema(dataSourcesToQuery), true) .forEach(item -> polledDataSourceSchema.put( item.getDatasource(), physicalDatasourceMetadataBuilder.build( @@ -121,7 +121,7 @@ public void refresh(final Set segmentsToRefresh, final Set da @Override public void rebuildDatasource(String dataSource) { - final DatasourceSchema druidTable = buildDruidTable(dataSource); + final DataSourceSchema druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 7f4041fb0b03..00a068a79950 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -11,6 +11,9 @@ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig @JsonProperty private long metadataSegmentPollPeriod = 60000; + @JsonProperty + private boolean awaitInitializationOnStart = true; + public boolean isMetadataSegmentCacheEnable() { return metadataSegmentCacheEnable; @@ -21,4 +24,9 @@ public long getMetadataSegmentPollPeriod() return metadataSegmentPollPeriod; } + @Override + public boolean isAwaitInitializationOnStart() + { + return awaitInitializationOnStart; + } } From 14baf19cb2bd15472e5cf2e10986d2c63a03e6b2 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 7 Sep 2023 13:47:55 +0530 Subject: [PATCH 08/36] cleanup and fix some tests --- .../timeline/SegmentStatusInCluster.java | 2 + .../timeline/SegmentStatusInClusterTest.java | 2 +- .../apache/druid/client/BrokerServerView.java | 13 +--- .../druid/client/CoordinatorServerView.java | 2 +- ...toryView.java => CoordinatorTimeline.java} | 14 +++- .../druid/client/InternalQueryConfig.java | 4 +- ...va => QueryableCoordinatorServerView.java} | 4 +- .../client/coordinator/CoordinatorClient.java | 5 +- .../metadata/AvailableSegmentMetadata.java | 12 +--- .../segment/metadata/DataSourceSchema.java | 37 +++++++++- .../server/http/DataSourcesResource.java | 8 +-- .../druid/server/http/MetadataResource.java | 33 ++++----- ...st.java => CoordinatorServerViewTest.java} | 8 +-- .../SegmentDataCacheConcurrencyTest.java | 4 +- .../metadata}/SegmentMetadataCacheCommon.java | 5 +- .../metadata}/SegmentMetadataCacheTest.java | 71 ++++++++----------- .../CuratorDruidCoordinatorTest.java | 6 +- .../server/http/DataSourcesResourceTest.java | 14 ++-- .../server/http/MetadataResourceTest.java | 13 ++-- .../server/http/ServersResourceTest.java | 4 +- .../org/apache/druid/cli/CliCoordinator.java | 14 ++-- .../schema/BrokerSegmentMetadataCache.java | 19 +++++ .../BrokerSegmentMetadataCacheConfig.java | 19 +++++ .../schema/BrokerSegmentMetadataView.java | 3 +- .../schema/DruidCalciteSchemaModule.java | 18 +---- .../PhysicalDatasourceMetadataBuilder.java | 19 +++++ .../org/apache/druid/sql/guice/SqlModule.java | 2 +- .../sql/calcite/schema/SystemSchemaTest.java | 24 ++++--- 28 files changed, 216 insertions(+), 163 deletions(-) rename server/src/main/java/org/apache/druid/client/{CoordinatorInventoryView.java => CoordinatorTimeline.java} (59%) rename server/src/main/java/org/apache/druid/client/{TimelineAwareCoordinatorServerView.java => QueryableCoordinatorServerView.java} (96%) rename server/src/test/java/org/apache/druid/client/{TimelineAwareCoordinatorServerViewTest.java => CoordinatorServerViewTest.java} (98%) rename {sql/src/test/java/org/apache/druid/sql/calcite/schema => server/src/test/java/org/apache/druid/segment/metadata}/SegmentDataCacheConcurrencyTest.java (99%) rename {sql/src/test/java/org/apache/druid/sql/calcite/schema => server/src/test/java/org/apache/druid/segment/metadata}/SegmentMetadataCacheCommon.java (94%) rename {sql/src/test/java/org/apache/druid/sql/calcite/schema => server/src/test/java/org/apache/druid/segment/metadata}/SegmentMetadataCacheTest.java (96%) diff --git a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java index bccfa764726b..a440a36923e6 100644 --- a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java +++ b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java @@ -32,6 +32,8 @@ *
  • the {@code DataSegment} object
    • *
  • overshadowed status of the segment
    • *
  • replication factor of the segment
    • + *
  • number of rows in the segment
    • + *
  • if the segment is realtime
    • * *

    * Objects of this class are used to sync the state of segments from the Coordinator to different services, typically the Broker. diff --git a/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java b/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java index 6e604f7ae9da..54190870e77c 100644 --- a/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java +++ b/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java @@ -76,7 +76,7 @@ private static SegmentStatusInCluster createSegmentForTest() 1 ); - return new SegmentStatusInCluster(dataSegment, OVERSHADOWED, REPLICATION_FACTOR, 1L, 0L, true); + return new SegmentStatusInCluster(dataSegment, OVERSHADOWED, REPLICATION_FACTOR, 10L, true); } @Test diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index 11624aad752a..efaf09232c35 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; import com.google.inject.Inject; import org.apache.druid.client.selector.QueryableDruidServer; @@ -433,12 +434,7 @@ public List getDruidServers() public Map getSegmentMetadata() { - return selectors; - } - - public Set getDatasourceNames() - { - return timelines.keySet(); + return ImmutableMap.copyOf(selectors); } Object getLock() @@ -455,9 +451,4 @@ Map getSelectors() { return selectors; } - - ConcurrentMap getClients() - { - return clients; - } } diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java index be4e90d9e600..3955fdbf7f1a 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorServerView.java @@ -26,7 +26,7 @@ * ServerView of coordinator for the state of segments being loaded in the cluster. */ @ManageLifecycle -public class CoordinatorServerView implements CoordinatorInventoryView +public class CoordinatorServerView implements CoordinatorTimeline { private static final Logger log = new Logger(CoordinatorServerView.class); diff --git a/server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java b/server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java similarity index 59% rename from server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java rename to server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java index a938c8864f1b..0998d5f79db3 100644 --- a/server/src/main/java/org/apache/druid/client/CoordinatorInventoryView.java +++ b/server/src/main/java/org/apache/druid/client/CoordinatorTimeline.java @@ -6,8 +6,20 @@ import java.util.Map; -public interface CoordinatorInventoryView extends InventoryView +/** + * Segment timeline maintained in the coordinator. + */ +public interface CoordinatorTimeline extends InventoryView { + /** + * Retrieve timeline for a dataSource. + */ VersionedIntervalTimeline getTimeline(DataSource dataSource); + + /** + * Server information for all segments in the timeline. + */ Map getSegmentLoadInfos(); + + } diff --git a/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java b/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java index 3c1d4d96022b..1098103ffe5c 100644 --- a/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java +++ b/server/src/main/java/org/apache/druid/client/InternalQueryConfig.java @@ -26,8 +26,8 @@ /** * This class contains configuration that internally generated Druid queries - * should add to their query payload. The runtime properties for this class - * have the prefix "druid.broker.internal.query.config." + * should add to their query payload. The runtime properties for this class have + * the prefix "druid.{service}.internal.query.config.". */ public class InternalQueryConfig { diff --git a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java similarity index 96% rename from server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java rename to server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index 88661386f0bc..ade760710def 100644 --- a/server/src/main/java/org/apache/druid/client/TimelineAwareCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -45,12 +45,12 @@ * ServerView of coordinator for the state of segments being loaded in the cluster. */ @ManageLifecycle -public class TimelineAwareCoordinatorServerView extends BrokerServerView implements CoordinatorInventoryView +public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline { private final FilteredServerInventoryView baseView; @Inject - public TimelineAwareCoordinatorServerView( + public QueryableCoordinatorServerView( final QueryToolChestWarehouse warehouse, final QueryWatcher queryWatcher, final @Smile ObjectMapper smileMapper, diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index ad72159bde02..66b739cf51cb 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -20,9 +20,9 @@ package org.apache.druid.client.coordinator; import com.google.common.util.concurrent.ListenableFuture; -import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; @@ -46,6 +46,9 @@ public interface CoordinatorClient */ ListenableFuture> fetchUsedSegments(String dataSource, List intervals); + /** + * Fetches schema for the given dataSources. + */ ListenableFuture> fetchDataSourceSchema(Set datasources); /** diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java index cd2f849438ee..a074f8e2bc9d 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/AvailableSegmentMetadata.java @@ -27,7 +27,7 @@ import java.util.Set; /** - * Immutable representation of RowSignature and other segment attributes needed by {@code SystemSchema.SegmentsTable} + * Immutable representation of RowSignature and other segment attributes. * This class contains the metadata of segments announced by historicals or ingestion tasks. */ public class AvailableSegmentMetadata @@ -158,14 +158,4 @@ public AvailableSegmentMetadata build() return new AvailableSegmentMetadata(this); } } - - @Override - public String toString() - { - return "AvailableSegmentMetadata{" + - "segmentId=" + segment.getId() + - ", isRealtime=" + isRealtime + - ", numRows=" + numRows + - '}'; - } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java index 15e87cf5d072..0528cfe3b1f8 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java @@ -4,9 +4,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.segment.column.RowSignature; +import java.util.Objects; + +/** + * Encapsulates schema information of a dataSource. + */ public class DataSourceSchema { - // dsinfo private final String datasource; private final RowSignature rowSignature; @@ -30,4 +34,35 @@ public RowSignature getRowSignature() { return rowSignature; } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DataSourceSchema that = (DataSourceSchema) o; + return Objects.equals(datasource, that.datasource) && Objects.equals( + rowSignature, + that.rowSignature + ); + } + + @Override + public int hashCode() + { + return Objects.hash(datasource, rowSignature); + } + + @Override + public String toString() + { + return "DataSourceSchema{" + + "datasource='" + datasource + '\'' + + ", rowSignature=" + rowSignature + + '}'; + } } diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index ea92c4e8471a..1cbaa92ae231 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -32,7 +32,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; -import org.apache.druid.client.CoordinatorInventoryView; +import org.apache.druid.client.CoordinatorTimeline; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableSegmentLoadInfo; import org.apache.druid.client.SegmentLoadInfo; @@ -102,7 +102,7 @@ public class DataSourcesResource private static final Logger log = new Logger(DataSourcesResource.class); private static final long DEFAULT_LOADSTATUS_INTERVAL_OFFSET = 14 * 24 * 60 * 60 * 1000; - private final CoordinatorInventoryView serverInventoryView; + private final CoordinatorTimeline serverInventoryView; private final SegmentsMetadataManager segmentsMetadataManager; private final MetadataRuleManager metadataRuleManager; private final OverlordClient overlordClient; @@ -111,7 +111,7 @@ public class DataSourcesResource @Inject public DataSourcesResource( - CoordinatorInventoryView serverInventoryView, + CoordinatorTimeline coordinatorTimeline, SegmentsMetadataManager segmentsMetadataManager, MetadataRuleManager metadataRuleManager, @Nullable OverlordClient overlordClient, @@ -119,7 +119,7 @@ public DataSourcesResource( DruidCoordinator coordinator ) { - this.serverInventoryView = serverInventoryView; + this.serverInventoryView = coordinatorTimeline; this.segmentsMetadataManager = segmentsMetadataManager; this.metadataRuleManager = metadataRuleManager; this.overlordClient = overlordClient; diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 5b70be95b5b4..1de89a11de73 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -214,8 +214,8 @@ private Response getAllUsedSegmentsWithAdditionalDetails( Integer replicationFactor = isOvershadowed ? (Integer) 0 : coordinator.getReplicationFactor(segment.getId()); segmentAlreadySeen.add(segment.getId()); - return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, 20L, true); - }).peek(v -> log.info("peeking into first stream segmentseen [%s], id [%s]", segmentAlreadySeen, v.getDataSegment().getId())); + return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, numRows, true); + }); Stream finalSegments = segmentStatus; @@ -225,29 +225,17 @@ private Response getAllUsedSegmentsWithAdditionalDetails( .getSegmentMetadataSnapshot() .values() .stream() - .peek(v -> log.info( - "peeking into second stream (first) segmentseen [%s], id [%s]", - segmentAlreadySeen, - v.getSegment().getId() - )) .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment() .getId())) - .map(availableSegmentMetadata -> { - return new SegmentStatusInCluster( - availableSegmentMetadata.getSegment(), - false, - (int) availableSegmentMetadata.getNumReplicas(), - /**availableSegmentMetadata.getNumRows(), **/30L, - availableSegmentMetadata.isRealtime() != 0 - ); - }) - .peek(v -> log.info("peeking into second stream (second) segmentseen [%s], id [%s]", - segmentAlreadySeen, - v.getDataSegment().getId() + .map(availableSegmentMetadata -> new SegmentStatusInCluster( + availableSegmentMetadata.getSegment(), + false, + (int) availableSegmentMetadata.getNumReplicas(), + availableSegmentMetadata.getNumRows(), + availableSegmentMetadata.isRealtime() != 0 )); - finalSegments = Stream.concat(segmentStatus, realtimeSegmentStatus) - .peek(v -> log.info("combined stream element %s", v)); + finalSegments = Stream.concat(segmentStatus, realtimeSegmentStatus); } final Function> raGenerator = segment -> Collections @@ -359,6 +347,9 @@ public Response getUsedSegment( return Response.status(Response.Status.NOT_FOUND).build(); } + /** + * Return schema for the given dataSources. + */ @POST @Path("/dataSourceSchema") @Produces(MediaType.APPLICATION_JSON) diff --git a/server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java similarity index 98% rename from server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java rename to server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java index 190a41e53f9e..1bca2a0690af 100644 --- a/server/src/test/java/org/apache/druid/client/TimelineAwareCoordinatorServerViewTest.java +++ b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java @@ -52,7 +52,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; -public class TimelineAwareCoordinatorServerViewTest extends CuratorTestBase +public class CoordinatorServerViewTest extends CuratorTestBase { private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; @@ -63,9 +63,9 @@ public class TimelineAwareCoordinatorServerViewTest extends CuratorTestBase private CountDownLatch segmentRemovedLatch; private BatchServerInventoryView baseView; - private TimelineAwareCoordinatorServerView overlordServerView; + private CoordinatorServerView overlordServerView; - public TimelineAwareCoordinatorServerViewTest() + public CoordinatorServerViewTest() { jsonMapper = TestHelper.makeJsonMapper(); zkPathsConfig = new ZkPathsConfig(); @@ -332,7 +332,7 @@ public CallbackAction segmentViewInitialized() } }; - overlordServerView = new TimelineAwareCoordinatorServerView( + overlordServerView = new CoordinatorServerView( baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter() diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java similarity index 99% rename from sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java rename to server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index d34bed3ccf73..dbc104eafa2b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.schema; +package org.apache.druid.segment.metadata; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; @@ -42,9 +42,7 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.metadata.PhysicalDatasourceMetadata; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; import org.apache.druid.query.TableDataSource; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java similarity index 94% rename from sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java rename to server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 97534f2f81d9..acf2c6b76c9d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.schema; +package org.apache.druid.segment.metadata; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -33,9 +33,6 @@ import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; -import org.apache.druid.sql.calcite.util.CalciteTestBase; -import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.easymock.EasyMock; import org.junit.AfterClass; import org.junit.Before; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java similarity index 96% rename from sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java rename to server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index 2cf08d7d6b71..5f1fccbd15f3 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.schema; +package org.apache.druid.segment.metadata; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; @@ -26,10 +26,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.data.input.InputRow; @@ -40,8 +36,6 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; -import org.apache.druid.segment.metadata.AvailableSegmentMetadata; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryContexts; @@ -72,13 +66,6 @@ import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.NoopEscalator; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; -import org.apache.druid.sql.calcite.table.DatasourceTable; -import org.apache.druid.sql.calcite.table.DruidTable; -import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; -import org.apache.druid.sql.calcite.util.TestDataBuilder; -import org.apache.druid.sql.calcite.util.TestServerInventoryView; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.DataSegment.PruneSpecsHolder; import org.apache.druid.timeline.SegmentId; @@ -133,7 +120,7 @@ public void setUp() throws Exception .withRollup(false) .build() ) - .rows(ROWS1) + .rows(SegmentMetadataCacheCommon.ROWS1) .buildMMappedIndex(); final QueryableIndex index2 = IndexBuilder.create() @@ -145,7 +132,7 @@ public void setUp() throws Exception .withRollup(false) .build() ) - .rows(ROWS2) + .rows(SegmentMetadataCacheCommon.ROWS2) .buildMMappedIndex(); final InputRowSchema rowSchema = new InputRowSchema( @@ -219,7 +206,7 @@ public void setUp() throws Exception .rows(autoRows2) .buildMMappedIndex(); - walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( + walker = new SpecificSegmentsQuerySegmentWalker(SegmentMetadataCacheCommon.conglomerate).add( DataSegment.builder() .dataSource(CalciteTests.DATASOURCE1) .interval(Intervals.of("2000/P1Y")) @@ -285,14 +272,14 @@ public void setUp() throws Exception public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { - return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); + return buildSchemaMarkAndTableLatch(SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT); } public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException { Preconditions.checkState(runningSchema == null); runningSchema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory( @@ -330,14 +317,14 @@ public SegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedE { Preconditions.checkState(runningSchema == null); runningSchema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory( ImmutableSet.of(globalTableJoinable), ImmutableMap.of(globalTableJoinable.getClass(), GlobalTableDataSource.class) ), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -704,11 +691,11 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -747,11 +734,11 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -794,11 +781,11 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -838,11 +825,11 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -879,11 +866,11 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -937,11 +924,11 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int CountDownLatch addSegmentLatch = new CountDownLatch(2); CountDownLatch removeSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -998,11 +985,11 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr String datasource = "serverSegmentRemoveTest"; CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -1033,11 +1020,11 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -1081,11 +1068,11 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -1327,7 +1314,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception ImmutableSet.of(globalTableJoinable), ImmutableMap.of(globalTableJoinable.getClass(), GlobalTableDataSource.class) ), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), internalQueryConfig, new NoopServiceEmitter() @@ -1459,11 +1446,11 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), serverView, segmentManager, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - SEGMENT_CACHE_CONFIG_DEFAULT, + SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), emitter diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java index 647eab9c6c45..b9b6fc8516fe 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java @@ -30,7 +30,7 @@ import org.apache.curator.utils.ZKPaths; import org.apache.druid.client.BatchServerInventoryView; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; -import org.apache.druid.client.Alpha; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DataSourcesSnapshot; import org.apache.druid.client.DruidServer; import org.apache.druid.client.ImmutableDruidDataSource; @@ -97,7 +97,7 @@ public class CuratorDruidCoordinatorTest extends CuratorTestBase private static final long COORDINATOR_PERIOD = 100; private BatchServerInventoryView baseView; - private Alpha serverView; + private CoordinatorServerView serverView; private CountDownLatch segmentViewInitLatch; /** * The following two fields are changed during {@link #testMoveSegment()}, the change might not be visible from the @@ -405,7 +405,7 @@ public CallbackAction segmentViewInitialized() } }; - serverView = new Alpha(baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter()); + serverView = new CoordinatorServerView(baseView, new CoordinatorSegmentWatcherConfig(), new NoopServiceEmitter()); baseView.start(); diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java index 5cdf1d5b73b9..551151458546 100644 --- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java @@ -28,7 +28,7 @@ import com.google.common.util.concurrent.Futures; import it.unimi.dsi.fastutil.objects.Object2LongMap; import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap; -import org.apache.druid.client.Alpha; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; import org.apache.druid.client.ImmutableDruidDataSource; @@ -83,7 +83,7 @@ public class DataSourcesResourceTest { - private Alpha inventoryView; + private CoordinatorServerView inventoryView; private DruidServer server; private List listDataSources; private List dataSegmentList; @@ -94,7 +94,7 @@ public class DataSourcesResourceTest public void setUp() { request = EasyMock.createStrictMock(HttpServletRequest.class); - inventoryView = EasyMock.createStrictMock(Alpha.class); + inventoryView = EasyMock.createStrictMock(CoordinatorServerView.class); server = EasyMock.niceMock(DruidServer.class); dataSegmentList = new ArrayList<>(); dataSegmentList.add( @@ -1302,7 +1302,7 @@ public void testGetDatasourceLoadstatusDefault() // Test when datasource fully loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(completedLoadInfoMap).once(); + EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1318,7 +1318,7 @@ public void testGetDatasourceLoadstatusDefault() // Test when datasource half loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(halfLoadedInfoMap).once(); + EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1381,7 +1381,7 @@ public void testGetDatasourceLoadstatusSimple() // Test when datasource fully loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(completedLoadInfoMap).once(); + EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(completedLoadInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); @@ -1397,7 +1397,7 @@ public void testGetDatasourceLoadstatusSimple() // Test when datasource half loaded EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq("datasource1"), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean())) .andReturn(Optional.of(segments)).once(); - EasyMock.expect(inventoryView.getLoadedSegmentIds()).andReturn(halfLoadedInfoMap).once(); + EasyMock.expect(inventoryView.getSegmentLoadInfos()).andReturn(halfLoadedInfoMap).once(); EasyMock.replay(segmentsMetadataManager, inventoryView); dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null); diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index 717db5f03caa..c0978e80c367 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -101,22 +101,23 @@ public void setUp() segmentsMetadataManager, storageCoordinator, AuthTestUtils.TEST_AUTHORIZER_MAPPER, - coordinator + coordinator, + null ); } @Test public void testGetAllSegmentsWithOvershadowedStatus() { - Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus"); + Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus", null); final List resultList = extractSegmentStatusList(response); Assert.assertEquals(resultList.size(), 4); - Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 5L, 0L, true), resultList.get(0)); - Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 5L, 0L, true), resultList.get(1)); - Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, 5L, 0L, true), resultList.get(2)); + Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 5L, true), resultList.get(0)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 5L, true), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, 5L, true), resultList.get(2)); // Replication factor should be 0 as the segment is overshadowed - Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, 5L, 0L, true), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, 5L, true), resultList.get(3)); } @Test diff --git a/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java b/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java index 951070557d7d..f96d2a3e74c8 100644 --- a/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/ServersResourceTest.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import org.apache.druid.client.Alpha; +import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.DruidServer; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; @@ -53,7 +53,7 @@ public void setUp() .build(); dummyServer.addDataSegment(segment); - Alpha inventoryView = EasyMock.createMock(Alpha.class); + CoordinatorServerView inventoryView = EasyMock.createMock(CoordinatorServerView.class); EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(dummyServer)).anyTimes(); EasyMock.expect(inventoryView.getInventoryValue(dummyServer.getName())).andReturn(dummyServer).anyTimes(); EasyMock.replay(inventoryView); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index c9b23df64fb2..652673e4239f 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -41,9 +41,9 @@ import org.apache.druid.client.CoordinatorSegmentWatcherConfig; import org.apache.druid.client.CoordinatorServerView; import org.apache.druid.client.HttpServerInventoryViewResource; -import org.apache.druid.client.CoordinatorInventoryView; +import org.apache.druid.client.CoordinatorTimeline; import org.apache.druid.client.InternalQueryConfig; -import org.apache.druid.client.TimelineAwareCoordinatorServerView; +import org.apache.druid.client.QueryableCoordinatorServerView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.Coordinator; import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig; @@ -232,7 +232,7 @@ public void configure(Binder binder) if (isSegmentMetadataCacheEnabled()) { binder.install(new SegmentMetadataCacheModule()); } else { - binder.bind(CoordinatorInventoryView.class).to(CoordinatorServerView.class).in(LazySingleton.class); + binder.bind(CoordinatorTimeline.class).to(CoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, CoordinatorServerView.class); binder.bind(SegmentMetadataCache.class).toProvider(Providers.of(null)); } @@ -541,10 +541,10 @@ public void configure(Binder binder) binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); binder.bind(CachingClusteredClient.class).in(LazySingleton.class); - binder.bind(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); - binder.bind(CoordinatorInventoryView.class).to(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); - binder.bind(TimelineServerView.class).to(TimelineAwareCoordinatorServerView.class).in(LazySingleton.class); - LifecycleModule.register(binder, TimelineAwareCoordinatorServerView.class); + binder.bind(QueryableCoordinatorServerView.class).in(LazySingleton.class); + binder.bind(CoordinatorTimeline.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); + binder.bind(TimelineServerView.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); + LifecycleModule.register(binder, QueryableCoordinatorServerView.class); LifecycleModule.register(binder, SegmentMetadataCache.class); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 8f3cd67abe66..b55dc855572a 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.sql.calcite.schema; import com.google.common.collect.Sets; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 00a068a79950..834b2d7079e1 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.sql.calcite.schema; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index 9eeafcd4f8c4..fc20c38d00a4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -61,7 +61,7 @@ /** * This class polls the Coordinator in background to keep the latest published segments. - * Provides {@link #getSegmentMetadata()} for others to get segments in metadata store. + * Provides {@link #getSegmentTableView()} for sys segments table view. * * This class polls the data from {@link SegmentsMetadataManager} object in the memory of the * currently leading Coordinator via HTTP queries. @@ -284,6 +284,7 @@ private ImmutableSortedSet fetchSegmentMetadata() return builder.build(); } + // Note that coordinator must be up to get segments private JsonParserIterator querySegmentMetadata( Set watchedDataSources diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java index 3003b99fbe1c..132f2cb92590 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java @@ -38,16 +38,8 @@ public class DruidCalciteSchemaModule implements Module { private static final String DRUID_SCHEMA_NAME = "druid"; private static final String INFORMATION_SCHEMA_NAME = "INFORMATION_SCHEMA"; - private static final String SEGMENT_METADATA_CACHE_ENABLED = "druid.sql.planner.segmentMetadataCacheEnabled"; static final String INCOMPLETE_SCHEMA = "INCOMPLETE_SCHEMA"; - private final Properties properties; - - public DruidCalciteSchemaModule(Properties properties) - { - this.properties = properties; - } - @Override public void configure(Binder binder) { @@ -60,9 +52,8 @@ public void configure(Binder binder) .in(Scopes.SINGLETON); // BrokerSegmentMetadataCache needs to listen to changes for incoming segments - if (isSegmentMetadataCacheEnabled()) { - LifecycleModule.register(binder, BrokerSegmentMetadataCache.class); - } + LifecycleModule.register(binder, BrokerSegmentMetadataCache.class); + binder.bind(DruidSchema.class).in(Scopes.SINGLETON); binder.bind(SystemSchema.class).in(Scopes.SINGLETON); binder.bind(InformationSchema.class).in(Scopes.SINGLETON); @@ -82,9 +73,4 @@ private DruidSchemaCatalog getRootSchema(@Named(INCOMPLETE_SCHEMA) DruidSchemaCa rootSchema.getRootSchema().add(INFORMATION_SCHEMA_NAME, informationSchema); return rootSchema; } - - private boolean isSegmentMetadataCacheEnabled() - { - return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_ENABLED, "true")); - } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index c9760aeb0cf4..5dfbfb664440 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.sql.calcite.schema; import com.google.inject.Inject; diff --git a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java index 03a774a05b61..56d0d2d5d41f 100644 --- a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java +++ b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java @@ -104,7 +104,7 @@ public void configure(Binder binder) binder.bind(TableDefnRegistry.class).in(LazySingleton.class); - binder.install(new DruidCalciteSchemaModule(props)); + binder.install(new DruidCalciteSchemaModule()); binder.install(new CalcitePlannerModule()); binder.install(new SqlAggregationModule()); binder.install(new DruidViewModule()); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index 52439c33d6e9..659be26029e4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -42,6 +42,7 @@ import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.client.TimelineServerView; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.InputRow; import org.apache.druid.discovery.DataNodeService; @@ -252,15 +253,17 @@ public void setUp() throws Exception .add(segment2, index2) .add(segment3, index3); - SegmentMetadataCache cache = new SegmentMetadataCache( + BrokerSegmentMetadataCache cache = new BrokerSegmentMetadataCache( CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), new TestServerInventoryView(walker.getSegments(), realtimeSegments), - new SegmentManager(EasyMock.createMock(SegmentLoader.class)), - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), - new NoopServiceEmitter() + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder( + new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), + new SegmentManager(EasyMock.createMock(SegmentLoader.class))), + new NoopCoordinatorClient() ); cache.start(); cache.awaitInitialization(); @@ -269,7 +272,6 @@ public void setUp() throws Exception druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class); serverInventoryView = EasyMock.createMock(FilteredServerInventoryView.class); schema = new SystemSchema( - druidSchema, metadataView, serverView, serverInventoryView, @@ -566,13 +568,13 @@ public void testGetTableMap() @Test public void testSegmentsTable() throws Exception { - final SegmentsTable segmentsTable = new SegmentsTable(druidSchema, metadataView, new ObjectMapper(), authMapper); + final SegmentsTable segmentsTable = new SegmentsTable(metadataView, new ObjectMapper(), authMapper); final Set publishedSegments = new HashSet<>(Arrays.asList( - new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 2L, 0L, true), - new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 2L, 0L, true), - new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 2L, 0L, true), - new SegmentStatusInCluster(segment1, true, 2, 2L, 0L, true), - new SegmentStatusInCluster(segment2, false, 0, 2L, 0L, true) + new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 2L, true), + new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 2L, true), + new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 2L, true), + new SegmentStatusInCluster(segment1, true, 2, 2L, true), + new SegmentStatusInCluster(segment2, false, 0, 2L, true) )); EasyMock.expect(metadataView.getSegmentMetadata()).andReturn(publishedSegments.iterator()).once(); From 857f056a130736112b0a14d12d7dd07da18397ba Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 7 Sep 2023 20:31:37 +0530 Subject: [PATCH 09/36] Port tests and test build failure --- ...ruidSchemaInternRowSignatureBenchmark.java | 8 - .../druid/benchmark/query/SqlBenchmark.java | 2 +- .../query/SqlExpressionBenchmark.java | 2 +- .../query/SqlNestedDataBenchmark.java | 2 +- .../benchmark/query/SqlVsNativeBenchmark.java | 2 +- ...ressedBigDecimalSqlAggregatorTestBase.java | 2 +- .../sql/TDigestSketchSqlAggregatorTest.java | 2 +- .../hll/sql/HllSketchSqlAggregatorTest.java | 2 +- .../sql/DoublesSketchSqlAggregatorTest.java | 2 +- .../sql/ThetaSketchSqlAggregatorTest.java | 2 +- ...ArrayOfDoublesSketchSqlAggregatorTest.java | 2 +- .../sql/BloomFilterSqlAggregatorTest.java | 2 +- ...etsHistogramQuantileSqlAggregatorTest.java | 2 +- .../sql/QuantileSqlAggregatorTest.java | 2 +- .../apache/druid/msq/test/MSQTestBase.java | 2 +- .../msq/test/MSQTestControllerContext.java | 2 +- .../sql/VarianceSqlAggregatorTest.java | 2 +- .../apache/druid/client/BrokerServerView.java | 31 +-- .../QueryableCoordinatorServerView.java | 6 +- .../metadata/SegmentMetadataCache.java | 43 ++- .../metadata/SegmentMetadataCacheConfig.java | 4 +- .../coordinator/NoopCoordinatorClient.java | 8 + .../SegmentDataCacheConcurrencyTest.java | 20 +- .../metadata/SegmentMetadataCacheCommon.java | 126 +++++---- .../SegmentMetadataCacheConfigTest.java | 23 +- .../metadata/SegmentMetadataCacheTest.java | 246 +++--------------- .../metadata/TestTimelineServerView.java | 8 +- .../SpecificSegmentsQuerySegmentWalker.java | 5 +- .../schema/BrokerSegmentMetadataCache.java | 12 +- .../BrokerSegmentMetadataCacheConfig.java | 5 + .../schema/BrokerSegmentMetadataView.java | 11 +- .../apache/druid/sql/SqlStatementTest.java | 2 +- .../sql/avatica/DruidAvaticaHandlerTest.java | 2 +- .../druid/sql/avatica/DruidStatementTest.java | 2 +- .../sql/calcite/BaseCalciteQueryTest.java | 2 +- .../calcite/CalciteNestedDataQueryTest.java | 2 +- .../sql/calcite/DrillWindowQueryTest.java | 2 +- .../SqlVectorizedExpressionSanityTest.java | 2 +- .../schema/DruidSchemaNoDataInitTest.java | 19 +- .../sql/calcite/schema/SystemSchemaTest.java | 29 ++- .../druid/sql/calcite/util/CalciteTests.java | 37 ++- .../sql/calcite/util/QueryFrameworkUtils.java | 42 +-- .../sql/calcite/util/SqlTestFramework.java | 1 + .../sql/calcite/util/TestDataBuilder.java | 1 + .../druid/sql/http/SqlResourceTest.java | 2 +- 45 files changed, 288 insertions(+), 445 deletions(-) rename {sql/src/test/java/org/apache/druid/sql/calcite/planner => server/src/test/java/org/apache/druid/segment/metadata}/SegmentMetadataCacheConfigTest.java (75%) rename sql/src/test/java/org/apache/druid/sql/calcite/util/TestServerInventoryView.java => server/src/test/java/org/apache/druid/segment/metadata/TestTimelineServerView.java (96%) rename {sql/src/test/java/org/apache/druid/sql/calcite/util => server/src/test/java/org/apache/druid/server}/SpecificSegmentsQuerySegmentWalker.java (97%) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index e87efb348079..4d2d11ba1f3f 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -76,9 +76,6 @@ private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCac public SegmentMetadataCacheForBenchmark( final QueryLifecycleFactory queryLifecycleFactory, final TimelineServerView serverView, - final SegmentManager segmentManager, - final JoinableFactory joinableFactory, - final PlannerConfig config, final Escalator escalator, final InternalQueryConfig internalQueryConfig ) @@ -86,8 +83,6 @@ public SegmentMetadataCacheForBenchmark( super( queryLifecycleFactory, serverView, - segmentManager, - joinableFactory, SegmentMetadataCacheConfig.create(), escalator, internalQueryConfig, @@ -177,9 +172,6 @@ public void setup() EasyMock.mock(QueryLifecycleFactory.class), EasyMock.mock(TimelineServerView.class), null, - null, - EasyMock.mock(PlannerConfig.class), - null, null ); DruidServerMetadata serverMetadata = new DruidServerMetadata( diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java index 1172d823e024..d340f10e35e3 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java @@ -63,7 +63,7 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 6f96a2cde568..09b5a74d4380 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -48,7 +48,7 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java index fe5dfff0c846..138de4ea3132 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java @@ -57,7 +57,7 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java index de3db00accf5..8c709a7457f9 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java @@ -49,7 +49,7 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java b/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java index 21dc9d9f0323..b6aa0d866e5d 100644 --- a/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java +++ b/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java @@ -44,7 +44,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java index f42cade76737..dd15a6defac2 100644 --- a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java +++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java @@ -49,7 +49,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java index 349f1a57d1c0..c08bc8f04345 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java @@ -76,7 +76,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.DataSegment; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java index c2d81cede8ce..68f53687f892 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java @@ -60,7 +60,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java index 3946ce558b19..b4d7e29661cb 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java @@ -62,7 +62,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.DataSegment; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java index a240f89bdcc8..f262accfef94 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java @@ -49,7 +49,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java index 1c77f0986e11..30b76e8602b5 100644 --- a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java +++ b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java @@ -50,7 +50,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java index d0a12ab2f2d5..fea3d871e938 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java @@ -52,7 +52,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java index 9e36def53cb8..0c81b4c40fcb 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java @@ -51,7 +51,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java index 4e00fd657ac5..ec97a34dfff0 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java @@ -162,7 +162,7 @@ import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.LookylooModule; import org.apache.druid.sql.calcite.util.QueryFrameworkUtils; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.SqlTestFramework; import org.apache.druid.sql.calcite.view.InProcessViewManager; import org.apache.druid.sql.guice.SqlBindings; diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestControllerContext.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestControllerContext.java index 2ee2207fd83a..c1a33a4665e4 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestControllerContext.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestControllerContext.java @@ -48,7 +48,7 @@ import org.apache.druid.msq.util.MultiStageQueryContext; import org.apache.druid.query.QueryContext; import org.apache.druid.server.DruidNode; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java index fe68b2737ef3..23d404354701 100644 --- a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java +++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java @@ -58,7 +58,7 @@ import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index efaf09232c35..d02f04b46d7e 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -71,11 +71,7 @@ public class BrokerServerView implements TimelineServerView { private static final Logger log = new Logger(BrokerServerView.class); - private final Object lock = new Object(); - private final ConcurrentMap clients = new ConcurrentHashMap<>(); - private final Map selectors = new HashMap<>(); - private final Map> timelines = new HashMap<>(); private final ConcurrentMap timelineCallbacks = new ConcurrentHashMap<>(); private final QueryToolChestWarehouse warehouse; @@ -87,9 +83,14 @@ public class BrokerServerView implements TimelineServerView private final ServiceEmitter emitter; private final BrokerSegmentWatcherConfig segmentWatcherConfig; private final Predicate> segmentFilter; - private final CountDownLatch initialized = new CountDownLatch(1); + protected final Object lock = new Object(); + + protected final Map selectors = new HashMap<>(); + + protected final Map> timelines = new HashMap<>(); + @Inject public BrokerServerView( final QueryToolChestWarehouse warehouse, @@ -431,24 +432,4 @@ public List getDruidServers() .map(queryableDruidServer -> queryableDruidServer.getServer().toImmutableDruidServer()) .collect(Collectors.toList()); } - - public Map getSegmentMetadata() - { - return ImmutableMap.copyOf(selectors); - } - - Object getLock() - { - return lock; - } - - Map> getTimelines() - { - return timelines; - } - - Map getSelectors() - { - return selectors; - } } diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index ade760710def..b5f8973519f9 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -75,9 +75,9 @@ public boolean isAwaitInitializationOnStart() public VersionedIntervalTimeline getTimeline(DataSource dataSource) { String table = Iterables.getOnlyElement(dataSource.getTableNames()); - synchronized (getLock()) { + synchronized (lock) { // build a new timeline? - VersionedIntervalTimeline timeline = getTimelines().get(table); + VersionedIntervalTimeline timeline = timelines.get(table); Collection x = timeline.iterateAllObjects(); VersionedIntervalTimeline newTimeline = new VersionedIntervalTimeline<>(Comparator.naturalOrder()); newTimeline.addAll(x.stream().map(v -> new VersionedIntervalTimeline.PartitionChunkEntry( @@ -90,7 +90,7 @@ public VersionedIntervalTimeline getTimeline(DataSource @Override public Map getSegmentLoadInfos() { - return CollectionUtils.mapValues(getSelectors(), ServerSelector::toSegmentLoadInfo); + return CollectionUtils.mapValues(selectors, ServerSelector::toSegmentLoadInfo); } @Override diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 3c20b4bcd779..56328fbc3dec 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -176,28 +176,10 @@ public class SegmentMetadataCache // For awaitInitialization. private final CountDownLatch initialized = new CountDownLatch(1); - /** - * This lock coordinates the access from multiple threads to those variables guarded by this lock. - * Currently, there are 2 threads that can access these variables. - * - * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. - * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceSchema} if necessary - * based on the information collected via timeline callbacks. - */ - private final Object lock = new Object(); - // All mutable segments. @GuardedBy("lock") private final TreeSet mutableSegments = new TreeSet<>(SEGMENT_ORDER); - // All dataSources that need tables regenerated. - @GuardedBy("lock") - private final Set dataSourcesNeedingRebuild = new HashSet<>(); - - // All segments that need to be refreshed. - @GuardedBy("lock") - private final TreeSet segmentsNeedingRefresh = new TreeSet<>(SEGMENT_ORDER); - // Configured context to attach to internally generated queries. private final InternalQueryConfig internalQueryConfig; @@ -214,6 +196,24 @@ public class SegmentMetadataCache */ private int totalSegments = 0; + /** + * This lock coordinates the access from multiple threads to those variables guarded by this lock. + * Currently, there are 2 threads that can access these variables. + * + * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. + * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceSchema} if necessary + * based on the information collected via timeline callbacks. + */ + protected final Object lock = new Object(); + + // All dataSources that need tables regenerated. + @GuardedBy("lock") + protected final Set dataSourcesNeedingRebuild = new HashSet<>(); + + // All segments that need to be refreshed. + @GuardedBy("lock") + protected final TreeSet segmentsNeedingRefresh = new TreeSet<>(SEGMENT_ORDER); + @Inject public SegmentMetadataCache( final QueryLifecycleFactory queryLifecycleFactory, @@ -281,17 +281,16 @@ public ServerView.CallbackAction serverSegmentRemoved( ); } - public void removeFromTable(String s) + protected void removeFromTable(String s) { tables.remove(s); } - public boolean tablesContains(String s) + protected boolean tablesContains(String s) { return tables.containsKey(s); } - private void startCacheExec() { cacheExec.submit( @@ -901,7 +900,7 @@ public Set getDataSourcesNeedingRebuild() } } - public Object getLock() + Object getLock() { return lock; } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index 80aba42ecddc..b1932485b1ad 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -9,7 +9,9 @@ public class SegmentMetadataCacheConfig private boolean awaitInitializationOnStart = false; @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); - private boolean enabled = false; + @JsonProperty + private final boolean enabled = false; + @JsonProperty private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); diff --git a/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java b/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java index bcaab5c255fb..b2cde828092e 100644 --- a/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java +++ b/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java @@ -22,10 +22,12 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; +import org.apache.druid.segment.metadata.DataSourceSchema; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; import java.util.List; +import java.util.Set; public class NoopCoordinatorClient implements CoordinatorClient { @@ -47,6 +49,12 @@ public ListenableFuture> fetchUsedSegments(String dataSource, throw new UnsupportedOperationException(); } + @Override + public ListenableFuture> fetchDataSourceSchema(Set datasources) + { + throw new UnsupportedOperationException(); + } + @Override public CoordinatorClient withRetryPolicy(ServiceRetryPolicy retryPolicy) { diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index dbc104eafa2b..92ad857476e1 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -21,8 +21,6 @@ import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.BrokerSegmentWatcherConfig; @@ -42,7 +40,6 @@ import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.http.client.HttpClient; -import org.apache.druid.metadata.PhysicalDatasourceMetadata; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; import org.apache.druid.query.TableDataSource; @@ -51,15 +48,12 @@ import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.NoopEscalator; -import org.apache.druid.sql.calcite.table.DatasourceTable; -import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.DataSegment.PruneSpecsHolder; import org.apache.druid.timeline.SegmentId; @@ -135,10 +129,8 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV throws InterruptedException, ExecutionException, TimeoutException { schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -146,7 +138,7 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV ) { @Override - DatasourceTable.PhysicalDatasourceMetadata buildDruidTable(final String dataSource) + public DataSourceSchema buildDruidTable(final String dataSource) { doInLock(() -> { try { @@ -245,10 +237,8 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() throws InterruptedException, ExecutionException, TimeoutException { schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -256,7 +246,7 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() ) { @Override - DatasourceTable.PhysicalDatasourceMetadata buildDruidTable(final String dataSource) + public DataSourceSchema buildDruidTable(final String dataSource) { doInLock(() -> { try { diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index acf2c6b76c9d..a391ba9fb74c 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -19,21 +19,28 @@ package org.apache.druid.segment.metadata; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.MapInputRowParser; +import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.io.Closer; -import org.apache.druid.query.DataSource; -import org.apache.druid.query.GlobalTableDataSource; +import org.apache.druid.query.DefaultGenericQueryMetricsFactory; +import org.apache.druid.query.DefaultQueryConfig; +import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunnerFactoryConglomerate; -import org.apache.druid.segment.join.JoinConditionAnalysis; -import org.apache.druid.segment.join.Joinable; -import org.apache.druid.segment.join.JoinableFactory; -import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; -import org.apache.druid.server.SegmentManager; -import org.easymock.EasyMock; +import org.apache.druid.server.log.TestRequestLogger; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.server.security.AuthConfig; +import org.apache.druid.server.security.AuthTestUtils; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -42,36 +49,59 @@ import java.io.IOException; import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.CountDownLatch; +import java.util.Map; -public abstract class SegmentMetadataCacheCommon extends CalciteTestBase +public abstract class SegmentMetadataCacheCommon { + public static final String DATASOURCE1 = "foo"; + public static final String DATASOURCE2 = "foo2"; + public static final String DATASOURCE3 = "numfoo"; + public static final String DATASOURCE4 = "foo4"; + public static final String DATASOURCE5 = "lotsocolumns"; + public static final String BROADCAST_DATASOURCE = "broadcast"; + public static final String FORBIDDEN_DATASOURCE = "forbiddenDatasource"; + public static final String SOME_DATASOURCE = "some_datasource"; + public static final String TIMESTAMP_COLUMN = "t"; + private static final InputRowSchema FOO_SCHEMA = new InputRowSchema( + new TimestampSpec(TIMESTAMP_COLUMN, "iso", null), + new DimensionsSpec( + DimensionsSpec.getDefaultSchemas(ImmutableList.of("dim1", "dim2", "dim3")) + ), + null + ); + static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); - static final List ROWS1 = ImmutableList.of( - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-02", "m1", "2.0", "dim1", "10.1")), - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-03", "m1", "3.0", "dim1", "2")) + final List ROWS1 = ImmutableList.of( + createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), + createRow(ImmutableMap.of("t", "2000-01-02", "m1", "2.0", "dim1", "10.1")), + createRow(ImmutableMap.of("t", "2000-01-03", "m1", "3.0", "dim1", "2")) ); - static final List ROWS2 = ImmutableList.of( - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-01", "m1", "4.0", "dim2", ImmutableList.of("a"))), - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-02", "m1", "5.0", "dim2", ImmutableList.of("abc"))), - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-03", "m1", "6.0")) + final List ROWS2 = ImmutableList.of( + createRow(ImmutableMap.of("t", "2001-01-01", "m1", "4.0", "dim2", ImmutableList.of("a"))), + createRow(ImmutableMap.of("t", "2001-01-02", "m1", "5.0", "dim2", ImmutableList.of("abc"))), + createRow(ImmutableMap.of("t", "2001-01-03", "m1", "6.0")) ); static QueryRunnerFactoryConglomerate conglomerate; static Closer resourceCloser; - CountDownLatch getDatasourcesLatch = new CountDownLatch(1); + static QueryToolChestWarehouse queryToolChestWarehouse; @BeforeClass public static void setUpClass() { resourceCloser = Closer.create(); conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); + queryToolChestWarehouse = new QueryToolChestWarehouse() + { + @Override + public > QueryToolChest getToolChest(final QueryType query) + { + return conglomerate.findFactory(query).getToolchest(); + } + }; } @AfterClass @@ -83,44 +113,32 @@ public static void tearDownClass() throws IOException @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - SegmentManager segmentManager; - Set segmentDataSourceNames; - Set joinableDataSourceNames; - JoinableFactory globalTableJoinable; @Before public void setUpCommon() { - segmentDataSourceNames = Sets.newConcurrentHashSet(); - joinableDataSourceNames = Sets.newConcurrentHashSet(); + } - segmentManager = new SegmentManager(EasyMock.createMock(SegmentLoader.class)) - { - @Override - public Set getDataSourceNames() - { - getDatasourcesLatch.countDown(); - return segmentDataSourceNames; - } - }; + InputRow createRow(final ImmutableMap map) + { + return MapInputRowParser.parse(FOO_SCHEMA, (Map) map); + } - globalTableJoinable = new JoinableFactory() - { - @Override - public boolean isDirectlyJoinable(DataSource dataSource) - { - return dataSource instanceof GlobalTableDataSource && - joinableDataSourceNames.contains(((GlobalTableDataSource) dataSource).getName()); - } + InputRow createRow(final ImmutableMap map, InputRowSchema inputRowSchema) + { + return MapInputRowParser.parse(inputRowSchema, (Map) map); + } - @Override - public Optional build( - DataSource dataSource, - JoinConditionAnalysis condition - ) - { - return Optional.empty(); - } - }; + QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) { + return new QueryLifecycleFactory( + queryToolChestWarehouse, + walker, + new DefaultGenericQueryMetricsFactory(), + new NoopServiceEmitter(), + new TestRequestLogger(), + new AuthConfig(), + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + Suppliers.ofInstance(new DefaultQueryConfig(ImmutableMap.of())) + ); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java similarity index 75% rename from sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java rename to server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java index 29de2639ae4a..16567719dffc 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/planner/SegmentMetadataCacheConfigTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.planner; +package org.apache.druid.segment.metadata; import com.google.common.collect.ImmutableList; import com.google.inject.Injector; @@ -37,21 +37,22 @@ */ public class SegmentMetadataCacheConfigTest { + private static final String CONFIG_BASE = "druid.coordinator.segmentMetadataCache"; + @Test public void testDefaultConfig() { final Injector injector = createInjector(); final JsonConfigProvider provider = JsonConfigProvider.of( - CalcitePlannerModule.CONFIG_BASE, + CONFIG_BASE, SegmentMetadataCacheConfig.class ); + final Properties properties = new Properties(); provider.inject(properties, injector.getInstance(JsonConfigurator.class)); final SegmentMetadataCacheConfig config = provider.get(); Assert.assertTrue(config.isAwaitInitializationOnStart()); - Assert.assertFalse(config.isMetadataSegmentCacheEnable()); Assert.assertEquals(Period.minutes(1), config.getMetadataRefreshPeriod()); - Assert.assertEquals(60_000, config.getMetadataSegmentPollPeriod()); Assert.assertEquals(new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); } @@ -60,24 +61,20 @@ public void testCustomizedConfig() { final Injector injector = createInjector(); final JsonConfigProvider provider = JsonConfigProvider.of( - CalcitePlannerModule.CONFIG_BASE, + CONFIG_BASE, SegmentMetadataCacheConfig.class ); final Properties properties = new Properties(); properties.setProperty( - CalcitePlannerModule.CONFIG_BASE + ".metadataColumnTypeMergePolicy", + CONFIG_BASE + ".metadataColumnTypeMergePolicy", "latestInterval" ); - properties.setProperty(CalcitePlannerModule.CONFIG_BASE + ".metadataRefreshPeriod", "PT2M"); - properties.setProperty(CalcitePlannerModule.CONFIG_BASE + ".metadataSegmentPollPeriod", "15000"); - properties.setProperty(CalcitePlannerModule.CONFIG_BASE + ".metadataSegmentCacheEnable", "true"); - properties.setProperty(CalcitePlannerModule.CONFIG_BASE + ".awaitInitializationOnStart", "false"); + properties.setProperty(CONFIG_BASE + ".metadataRefreshPeriod", "PT2M"); + properties.setProperty(CONFIG_BASE + ".awaitInitializationOnStart", "false"); provider.inject(properties, injector.getInstance(JsonConfigurator.class)); final SegmentMetadataCacheConfig config = provider.get(); Assert.assertFalse(config.isAwaitInitializationOnStart()); - Assert.assertTrue(config.isMetadataSegmentCacheEnable()); Assert.assertEquals(Period.minutes(2), config.getMetadataRefreshPeriod()); - Assert.assertEquals(15_000, config.getMetadataSegmentPollPeriod()); Assert.assertEquals( new SegmentMetadataCache.FirstTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy() @@ -89,7 +86,7 @@ private Injector createInjector() return GuiceInjectors.makeStartupInjectorWithModules( ImmutableList.of( binder -> { - JsonConfigProvider.bind(binder, CalcitePlannerModule.CONFIG_BASE, SegmentMetadataCacheConfig.class); + JsonConfigProvider.bind(binder, CONFIG_BASE, SegmentMetadataCacheConfig.class); } ) ); diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index 5f1fccbd15f3..dc18cfb79741 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -26,8 +26,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ImmutableDruidServer; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.DimensionsSpec; @@ -37,7 +37,6 @@ import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.query.DruidMetrics; -import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.aggregation.CountAggregatorFactory; @@ -55,11 +54,11 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryResponse; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -95,7 +94,7 @@ public class SegmentMetadataCacheTest extends SegmentMetadataCacheCommon private static final int WAIT_TIMEOUT_SECS = 6; private SpecificSegmentsQuerySegmentWalker walker; - private TestServerInventoryView serverView; + private TestTimelineServerView serverView; private List druidServers; private SegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); @@ -120,7 +119,7 @@ public void setUp() throws Exception .withRollup(false) .build() ) - .rows(SegmentMetadataCacheCommon.ROWS1) + .rows(ROWS1) .buildMMappedIndex(); final QueryableIndex index2 = IndexBuilder.create() @@ -132,7 +131,7 @@ public void setUp() throws Exception .withRollup(false) .build() ) - .rows(SegmentMetadataCacheCommon.ROWS2) + .rows(ROWS2) .buildMMappedIndex(); final InputRowSchema rowSchema = new InputRowSchema( @@ -141,7 +140,7 @@ public void setUp() throws Exception null ); final List autoRows1 = ImmutableList.of( - TestDataBuilder.createRow( + createRow( ImmutableMap.builder() .put("t", "2023-01-01T00:00Z") .put("numbery", 1.1f) @@ -154,7 +153,7 @@ public void setUp() throws Exception ) ); final List autoRows2 = ImmutableList.of( - TestDataBuilder.createRow( + createRow( ImmutableMap.builder() .put("t", "2023-01-02T00:00Z") .put("numbery", 1L) @@ -166,6 +165,7 @@ public void setUp() throws Exception rowSchema ) ); + final QueryableIndex indexAuto1 = IndexBuilder.create() .tmpDir(new File(tmpDir, "1")) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) @@ -208,7 +208,7 @@ public void setUp() throws Exception walker = new SpecificSegmentsQuerySegmentWalker(SegmentMetadataCacheCommon.conglomerate).add( DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE1) + .dataSource(DATASOURCE1) .interval(Intervals.of("2000/P1Y")) .version("1") .shardSpec(new LinearShardSpec(0)) @@ -217,7 +217,7 @@ public void setUp() throws Exception index1 ).add( DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE1) + .dataSource(DATASOURCE1) .interval(Intervals.of("2001/P1Y")) .version("1") .shardSpec(new LinearShardSpec(0)) @@ -226,7 +226,7 @@ public void setUp() throws Exception index2 ).add( DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE2) + .dataSource(DATASOURCE2) .interval(index2.getDataInterval()) .version("1") .shardSpec(new LinearShardSpec(0)) @@ -235,7 +235,7 @@ public void setUp() throws Exception index2 ).add( DataSegment.builder() - .dataSource(CalciteTests.SOME_DATASOURCE) + .dataSource(SOME_DATASOURCE) .interval(Intervals.of("2023-01-01T00Z/P1D")) .version("1") .shardSpec(new LinearShardSpec(1)) @@ -244,7 +244,7 @@ public void setUp() throws Exception indexAuto1 ).add( DataSegment.builder() - .dataSource(CalciteTests.SOME_DATASOURCE) + .dataSource(SOME_DATASOURCE) .interval(Intervals.of("2023-01-02T00Z/P1D")) .version("1") .shardSpec(new LinearShardSpec(1)) @@ -266,7 +266,7 @@ public void setUp() throws Exception PruneSpecsHolder.DEFAULT ); final List realtimeSegments = ImmutableList.of(segment1); - serverView = new TestServerInventoryView(walker.getSegments(), realtimeSegments); + serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); druidServers = serverView.getDruidServers(); } @@ -279,13 +279,8 @@ public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheCon { Preconditions.checkState(runningSchema == null); runningSchema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory( - ImmutableSet.of(globalTableJoinable), - ImmutableMap.of(globalTableJoinable.getClass(), GlobalTableDataSource.class) - ), config, new NoopEscalator(), new InternalQueryConfig(), @@ -293,9 +288,9 @@ public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheCon ) { @Override - protected DatasourceTable.PhysicalDatasourceMetadata buildDruidTable(String dataSource) + public DataSourceSchema buildDruidTable(String dataSource) { - DatasourceTable.PhysicalDatasourceMetadata table = super.buildDruidTable(dataSource); + DataSourceSchema table = super.buildDruidTable(dataSource); buildTableLatch.countDown(); return table; } @@ -317,13 +312,8 @@ public SegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedE { Preconditions.checkState(runningSchema == null); runningSchema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory( - ImmutableSet.of(globalTableJoinable), - ImmutableMap.of(globalTableJoinable.getClass(), GlobalTableDataSource.class) - ), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -364,68 +354,31 @@ public void tearDown() throws Exception public void testGetTableMap() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema.getDatasourceNames()); + Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); final Set tableNames = schema.getDatasourceNames(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), tableNames); + Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), tableNames); } @Test public void testSchemaInit() throws InterruptedException { SegmentMetadataCache schema2 = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema2.getDatasourceNames()); + Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema2.getDatasourceNames()); } @Test public void testGetTableMapFoo() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource("foo"); - final DruidTable fooTable = new DatasourceTable(fooDs); - final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); - final List fields = rowType.getFieldList(); - - Assert.assertEquals(6, fields.size()); - - Assert.assertEquals("__time", fields.get(0).getName()); - Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); - - Assert.assertEquals("dim2", fields.get(1).getName()); - Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(1).getType().getSqlTypeName()); - - Assert.assertEquals("m1", fields.get(2).getName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getSqlTypeName()); - - Assert.assertEquals("dim1", fields.get(3).getName()); - Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getSqlTypeName()); - - Assert.assertEquals("cnt", fields.get(4).getName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(4).getType().getSqlTypeName()); - - Assert.assertEquals("unique_dim1", fields.get(5).getName()); - Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); + final DataSourceSchema fooDs = schema.getDatasource("foo"); } @Test public void testGetTableMapFoo2() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource("foo2"); - final DruidTable fooTable = new DatasourceTable(fooDs); - final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); - final List fields = rowType.getFieldList(); - - Assert.assertEquals(3, fields.size()); - - Assert.assertEquals("__time", fields.get(0).getName()); - Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); - - Assert.assertEquals("dim2", fields.get(1).getName()); - Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(1).getType().getSqlTypeName()); - - Assert.assertEquals("m1", fields.get(2).getName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(2).getType().getSqlTypeName()); + final DataSourceSchema fooDs = schema.getDatasource("foo2"); } @Test @@ -442,41 +395,8 @@ public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePoli } } ); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource(CalciteTests.SOME_DATASOURCE); - final DruidTable table = new DatasourceTable(fooDs); - final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); - final List fields = rowType.getFieldList(); - - Assert.assertEquals(9, fields.size()); - - Assert.assertEquals("__time", fields.get(0).getName()); - Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); + final DataSourceSchema fooDs = schema.getDatasource(SOME_DATASOURCE); - Assert.assertEquals("numbery", fields.get(1).getName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(1).getType().getSqlTypeName()); - - Assert.assertEquals("numberyArrays", fields.get(2).getName()); - Assert.assertEquals(SqlTypeName.ARRAY, fields.get(2).getType().getSqlTypeName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getComponentType().getSqlTypeName()); - - Assert.assertEquals("stringy", fields.get(3).getName()); - Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getSqlTypeName()); - - Assert.assertEquals("array", fields.get(4).getName()); - Assert.assertEquals(SqlTypeName.ARRAY, fields.get(4).getType().getSqlTypeName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(4).getType().getComponentType().getSqlTypeName()); - - Assert.assertEquals("nested", fields.get(5).getName()); - Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); - - Assert.assertEquals("cnt", fields.get(6).getName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(6).getType().getSqlTypeName()); - - Assert.assertEquals("m1", fields.get(7).getName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(7).getType().getSqlTypeName()); - - Assert.assertEquals("unique_dim1", fields.get(8).getName()); - Assert.assertEquals(SqlTypeName.OTHER, fields.get(8).getType().getSqlTypeName()); } @Test @@ -485,42 +405,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource(CalciteTests.SOME_DATASOURCE); - final DruidTable table = new DatasourceTable(fooDs); - final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); - final List fields = rowType.getFieldList(); - - Assert.assertEquals(9, fields.size()); - - Assert.assertEquals("__time", fields.get(0).getName()); - Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); - - Assert.assertEquals("numbery", fields.get(1).getName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(1).getType().getSqlTypeName()); - - Assert.assertEquals("numberyArrays", fields.get(2).getName()); - Assert.assertEquals(SqlTypeName.ARRAY, fields.get(2).getType().getSqlTypeName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getComponentType().getSqlTypeName()); - - Assert.assertEquals("stringy", fields.get(3).getName()); - Assert.assertEquals(SqlTypeName.ARRAY, fields.get(3).getType().getSqlTypeName()); - Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getComponentType().getSqlTypeName()); - - Assert.assertEquals("array", fields.get(4).getName()); - Assert.assertEquals(SqlTypeName.ARRAY, fields.get(4).getType().getSqlTypeName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(4).getType().getComponentType().getSqlTypeName()); - - Assert.assertEquals("nested", fields.get(5).getName()); - Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); - - Assert.assertEquals("cnt", fields.get(6).getName()); - Assert.assertEquals(SqlTypeName.BIGINT, fields.get(6).getType().getSqlTypeName()); - - Assert.assertEquals("m1", fields.get(7).getName()); - Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(7).getType().getSqlTypeName()); - - Assert.assertEquals("unique_dim1", fields.get(8).getName()); - Assert.assertEquals(SqlTypeName.OTHER, fields.get(8).getType().getSqlTypeName()); + final DataSourceSchema fooDs = schema.getDatasource(SOME_DATASOURCE); } @@ -691,10 +576,8 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -734,10 +617,8 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -781,10 +662,8 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -825,10 +704,8 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -866,10 +743,8 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -924,10 +799,8 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int CountDownLatch addSegmentLatch = new CountDownLatch(2); CountDownLatch removeSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -985,10 +858,8 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr String datasource = "serverSegmentRemoveTest"; CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -1020,10 +891,8 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -1068,10 +937,8 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), @@ -1135,12 +1002,8 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte { SegmentMetadataCache schema3 = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DatasourceTable.PhysicalDatasourceMetadata fooTable = schema3.getDatasource("foo"); + DataSourceSchema fooTable = schema3.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertFalse(fooTable.isJoinable()); - Assert.assertFalse(fooTable.isBroadcast()); markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); @@ -1157,8 +1020,6 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte 100L, PruneSpecsHolder.DEFAULT ); - segmentDataSourceNames.add("foo"); - joinableDataSourceNames.add("foo"); serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); // wait for build twice @@ -1169,16 +1030,10 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte fooTable = schema3.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - Assert.assertTrue(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertTrue(fooTable.isJoinable()); - Assert.assertTrue(fooTable.isBroadcast()); // now remove it markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); - joinableDataSourceNames.remove("foo"); - segmentDataSourceNames.remove("foo"); serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1190,10 +1045,6 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte fooTable = schema3.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertFalse(fooTable.isJoinable()); - Assert.assertFalse(fooTable.isBroadcast()); } @Test @@ -1201,12 +1052,8 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw { SegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getDatasource("foo"); + DataSourceSchema fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertFalse(fooTable.isJoinable()); - Assert.assertFalse(fooTable.isBroadcast()); markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); @@ -1223,7 +1070,6 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw 100L, PruneSpecsHolder.DEFAULT ); - segmentDataSourceNames.add("foo"); serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1235,17 +1081,10 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - // Should not be a GlobalTableDataSource for now, because isGlobal is couple with joinability. Ideally this will be - // changed in the future and we should expect. - Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertTrue(fooTable.isBroadcast()); - Assert.assertFalse(fooTable.isJoinable()); // now remove it markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); - segmentDataSourceNames.remove("foo"); serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1257,10 +1096,6 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); - Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); - Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); - Assert.assertFalse(fooTable.isBroadcast()); - Assert.assertFalse(fooTable.isJoinable()); } /** @@ -1309,11 +1144,6 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception SegmentMetadataCache mySchema = new SegmentMetadataCache( factoryMock, serverView, - segmentManager, - new MapJoinableFactory( - ImmutableSet.of(globalTableJoinable), - ImmutableMap.of(globalTableJoinable.getClass(), GlobalTableDataSource.class) - ), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), internalQueryConfig, @@ -1446,10 +1276,8 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, SegmentMetadataCacheCommon.conglomerate), + getQueryLifecycleFactory(walker), serverView, - segmentManager, - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestServerInventoryView.java b/server/src/test/java/org/apache/druid/segment/metadata/TestTimelineServerView.java similarity index 96% rename from sql/src/test/java/org/apache/druid/sql/calcite/util/TestServerInventoryView.java rename to server/src/test/java/org/apache/druid/segment/metadata/TestTimelineServerView.java index d74b36739cfd..42c219665909 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestServerInventoryView.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/TestTimelineServerView.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.util; +package org.apache.druid.segment.metadata; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -44,7 +44,7 @@ /** * This class is used for testing and benchmark */ -public class TestServerInventoryView implements TimelineServerView +public class TestTimelineServerView implements TimelineServerView { private static final DruidServerMetadata DUMMY_SERVER = new DruidServerMetadata( "dummy", @@ -80,12 +80,12 @@ public class TestServerInventoryView implements TimelineServerView private List> segmentCallbackExecs = new ArrayList<>(); private List> timelineCallbackExecs = new ArrayList<>(); - public TestServerInventoryView(List segments) + public TestTimelineServerView(List segments) { this.segments.addAll(segments); } - public TestServerInventoryView(List segments, List realtimeSegments) + public TestTimelineServerView(List segments, List realtimeSegments) { this.segments.addAll(segments); this.realtimeSegments.addAll(realtimeSegments); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java b/server/src/test/java/org/apache/druid/server/SpecificSegmentsQuerySegmentWalker.java similarity index 97% rename from sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java rename to server/src/test/java/org/apache/druid/server/SpecificSegmentsQuerySegmentWalker.java index b43a65159567..c111cc9f7f4d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SpecificSegmentsQuerySegmentWalker.java +++ b/server/src/test/java/org/apache/druid/server/SpecificSegmentsQuerySegmentWalker.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.druid.sql.calcite.util; +package org.apache.druid.server; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; @@ -44,9 +44,6 @@ import org.apache.druid.segment.SegmentWrangler; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.join.JoinableFactoryWrapper; -import org.apache.druid.server.ClientQuerySegmentWalker; -import org.apache.druid.server.QueryScheduler; -import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.initialization.ServerConfig; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.VersionedIntervalTimeline; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index b55dc855572a..9fa0f3938b81 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -61,7 +61,7 @@ public class BrokerSegmentMetadataCache extends SegmentMetadataCache public BrokerSegmentMetadataCache( final QueryLifecycleFactory queryLifecycleFactory, final TimelineServerView serverView, - final SegmentMetadataCacheConfig config, + final BrokerSegmentMetadataCacheConfig config, final Escalator escalator, final InternalQueryConfig internalQueryConfig, final ServiceEmitter emitter, @@ -118,17 +118,17 @@ public void refresh(final Set segmentsToRefresh, final Set da // Refresh the segments. final Set refreshed = refreshSegments(segmentsToRefresh); - synchronized (getLock()) { + synchronized (lock) { // Add missing segments back to the refresh list. getSegmentsNeedingRefresh().addAll(Sets.difference(segmentsToRefresh, refreshed)); // Compute the list of dataSources to rebuild tables for. - dataSourcesToRebuild.addAll(getDataSourcesNeedingRebuild()); + dataSourcesToRebuild.addAll(dataSourcesNeedingRebuild); refreshed.forEach(segment -> dataSourcesToRebuild.add(segment.getDataSource())); // Remove those dataSource for which we received schema from the Coordinator. dataSourcesToRebuild.removeAll(polledDataSourceSchema.keySet()); - getDataSourcesNeedingRebuild().clear(); + dataSourcesNeedingRebuild.clear(); } // Rebuild the dataSources. @@ -163,13 +163,13 @@ public Set getDatasourceNames() } @Override - public void removeFromTable(String s) + protected void removeFromTable(String s) { tables.remove(s); } @Override - public boolean tablesContains(String s) + protected boolean tablesContains(String s) { return tables.containsKey(s); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 834b2d7079e1..2554753f79c3 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -33,6 +33,11 @@ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig @JsonProperty private boolean awaitInitializationOnStart = true; + public static BrokerSegmentMetadataCacheConfig create() + { + return new BrokerSegmentMetadataCacheConfig(); + } + public boolean isMetadataSegmentCacheEnable() { return metadataSegmentCacheEnable; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index fc20c38d00a4..9ef381b10b3f 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -90,8 +90,6 @@ public class BrokerSegmentMetadataView @MonotonicNonNull private volatile ImmutableSortedSet segmentMetadata = null; - private final BrokerServerView brokerServerView; - private final BrokerSegmentMetadataCache segmentMetadataCache; /** @@ -111,7 +109,6 @@ public BrokerSegmentMetadataView( final ObjectMapper objectMapper, final BrokerSegmentWatcherConfig segmentWatcherConfig, final BrokerSegmentMetadataCacheConfig config, - final BrokerServerView brokerServerView, final BrokerSegmentMetadataCache segmentMetadataCache ) { @@ -127,7 +124,6 @@ public BrokerSegmentMetadataView( this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() .expireAfterAccess(10, TimeUnit.MINUTES) .build(); - this.brokerServerView = brokerServerView; this.segmentMetadataCache = segmentMetadataCache; } @@ -167,7 +163,6 @@ protected Iterator getSegmentTableView() { final ImmutableSortedSet segments = getSegmentMetadata(); final Map availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); - final Map brokerSegmentMetadata = brokerServerView.getSegmentMetadata(); final List segmentsTableViews = new ArrayList<>(); Set seenSegments = new HashSet<>(); @@ -183,10 +178,6 @@ protected Iterator getSegmentTableView() numReplicas = availableSegmentMetadata.getNumReplicas(); numRows = availableSegmentMetadata.getNumRows(); isAvailable = 1L; - } else if (brokerSegmentMetadata.containsKey(segmentId)) { - ServerSelector serverSelector = brokerSegmentMetadata.get(segmentId); - numReplicas = serverSelector.getAllServers().size(); - isAvailable = 1L; } // Prefer numRows & realtime status info returned from Coordinator. @@ -244,7 +235,7 @@ private void pollSegmentMetadata() segmentMetadataCachePopulated.countDown(); } - ImmutableSortedSet getSegmentMetadata() + private ImmutableSortedSet getSegmentMetadata() { if (isMetadataSegmentCacheEnabled) { Uninterruptibles.awaitUninterruptibly(segmentMetadataCachePopulated); diff --git a/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java b/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java index 9aab1faeaaf4..cad98cd8409e 100644 --- a/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java +++ b/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java @@ -58,7 +58,7 @@ import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.http.SqlQuery; import org.easymock.EasyMock; import org.hamcrest.MatcherAssert; diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 1461c1e5ad8f..bcc2dddb4c9e 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -88,7 +88,7 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.guice.SqlModule; import org.eclipse.jetty.server.Server; import org.joda.time.DateTime; diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java index a1083ab30a7b..1df137e772c3 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java @@ -47,7 +47,7 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 3c73319898e4..ecd104d0b669 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -97,7 +97,7 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.SqlTestFramework; import org.apache.druid.sql.calcite.util.SqlTestFramework.Builder; import org.apache.druid.sql.calcite.util.SqlTestFramework.PlannerComponentSupplier; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index 406f94b46435..d366845bb9b2 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -69,7 +69,7 @@ import org.apache.druid.segment.virtual.NestedFieldVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.sql.calcite.filtration.Filtration; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java index 762ae621d1fd..27a099b1c479 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java @@ -44,7 +44,7 @@ import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMediumFactory; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.junit.Assert; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java index 2ac0574c2789..a439f1d19ea4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java @@ -48,7 +48,7 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java index c1a0d5047ea7..974edbd3a588 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java @@ -23,7 +23,6 @@ import com.google.common.collect.ImmutableSet; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.java.util.common.io.Closer; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; @@ -34,8 +33,8 @@ import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; -import org.apache.druid.sql.calcite.util.TestServerInventoryView; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.segment.metadata.TestTimelineServerView; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; @@ -44,25 +43,27 @@ public class DruidSchemaNoDataInitTest extends CalciteTestBase { - private static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create(); + private static final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create(); @Test public void testInitializationWithNoData() throws Exception { try (final Closer closer = Closer.create()) { final QueryRunnerFactoryConglomerate conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(closer); - final SegmentMetadataCache cache = new SegmentMetadataCache( + final BrokerSegmentMetadataCache cache = new BrokerSegmentMetadataCache( CalciteTests.createMockQueryLifecycleFactory( new SpecificSegmentsQuerySegmentWalker(conglomerate), conglomerate ), - new TestServerInventoryView(Collections.emptyList()), - new SegmentManager(EasyMock.createMock(SegmentLoader.class)), - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), + new TestTimelineServerView(Collections.emptyList()), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), - new NoopServiceEmitter() + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder( + new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), + new SegmentManager(EasyMock.createMock(SegmentLoader.class))), + null ); cache.start(); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index 659be26029e4..b17eb7713688 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -65,7 +65,6 @@ import org.apache.druid.java.util.http.client.response.HttpResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; @@ -97,16 +96,16 @@ import org.apache.druid.server.security.ResourceType; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable; +import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable.SegmentTableView; import org.apache.druid.sql.calcite.table.RowSignatures; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; -import org.apache.druid.sql.calcite.util.TestServerInventoryView; +import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.SegmentStatusInCluster; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.easymock.EasyMock; import org.jboss.netty.handler.codec.http.HttpResponse; @@ -134,7 +133,7 @@ public class SystemSchemaTest extends CalciteTestBase { - private static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create(); + private static final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create(); private static final List ROWS1 = ImmutableList.of( TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), @@ -255,14 +254,15 @@ public void setUp() throws Exception BrokerSegmentMetadataCache cache = new BrokerSegmentMetadataCache( CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - new TestServerInventoryView(walker.getSegments(), realtimeSegments), + new TestTimelineServerView(walker.getSegments(), realtimeSegments), SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), new PhysicalDatasourceMetadataBuilder( new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - new SegmentManager(EasyMock.createMock(SegmentLoader.class))), + new SegmentManager(EasyMock.createMock(SegmentLoader.class)) + ), new NoopCoordinatorClient() ); cache.start(); @@ -569,15 +569,16 @@ public void testGetTableMap() public void testSegmentsTable() throws Exception { final SegmentsTable segmentsTable = new SegmentsTable(metadataView, new ObjectMapper(), authMapper); - final Set publishedSegments = new HashSet<>(Arrays.asList( - new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 2L, true), - new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 2L, true), - new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 2L, true), - new SegmentStatusInCluster(segment1, true, 2, 2L, true), - new SegmentStatusInCluster(segment2, false, 0, 2L, true) + + final Set segmentTableViews = new HashSet<>(Arrays.asList( + new SegmentTableView(publishedCompactedSegment1, 1L, 0L, 1, 2, 2, true), + new SegmentTableView(publishedCompactedSegment2, 1L, 0L, 0, 2, 0, false), + new SegmentTableView(publishedUncompactedSegment3, 1L, 0L, 1, 2, 2, false), + new SegmentTableView(segment1, 1L, 0L, 1, 2, 2, true), + new SegmentTableView(segment2, 1L, 0L, 1, 2, 0, false) )); - EasyMock.expect(metadataView.getSegmentMetadata()).andReturn(publishedSegments.iterator()).once(); + EasyMock.expect(metadataView.getSegmentTableView()).andReturn(segmentTableViews.iterator()).once(); EasyMock.replay(client, request, responseHolder, responseHandler, metadataView); DataContext dataContext = createDataContext(Users.SUPER); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java index f182bd5dab58..3f5e9b14f5e9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -30,8 +31,11 @@ import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ServerInventoryView; import org.apache.druid.client.ServerView; +import org.apache.druid.client.TimelineServerView; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.client.indexing.NoopOverlordClient; import org.apache.druid.discovery.DiscoveryDruidNode; import org.apache.druid.discovery.DruidLeaderClient; @@ -49,10 +53,16 @@ import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.join.JoinableFactoryWrapper; +import org.apache.druid.segment.join.MapJoinableFactory; +import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.TestTimelineServerView; +import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.DruidNode; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.AuthConfig; @@ -68,14 +78,17 @@ import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.run.NativeSqlEngine; import org.apache.druid.sql.calcite.run.SqlEngine; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataView; import org.apache.druid.sql.calcite.schema.DruidSchema; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; +import org.apache.druid.sql.calcite.schema.PhysicalDatasourceMetadataBuilder; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.timeline.DataSegment; +import org.easymock.EasyMock; import org.joda.time.Duration; import javax.annotation.Nullable; @@ -330,7 +343,7 @@ public static DruidOperatorTable createOperatorTable() } public static SystemSchema createMockSystemSchema( - final DruidSchema druidSchema, + final QueryLifecycleFactory queryLifecycleFactory, final SpecificSegmentsQuerySegmentWalker walker, final AuthorizerMapper authorizerMapper ) @@ -370,15 +383,29 @@ public ListenableFuture findCurrentLeader() } }; + TimelineServerView timelineServerView = new TestTimelineServerView(walker.getSegments()); + return new SystemSchema( - druidSchema, new BrokerSegmentMetadataView( druidLeaderClient, getJsonMapper(), new BrokerSegmentWatcherConfig(), - SegmentMetadataCacheConfig.create() + BrokerSegmentMetadataCacheConfig.create(), + new BrokerSegmentMetadataCache( + queryLifecycleFactory, + timelineServerView, + BrokerSegmentMetadataCacheConfig.create(), + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder( + new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), + new SegmentManager(EasyMock.createMock(SegmentLoader.class)) + ), + new NoopCoordinatorClient() + ) ), - new TestServerInventoryView(walker.getSegments()), + timelineServerView, new FakeServerInventoryView(), authorizerMapper, druidLeaderClient, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index 1e2b9aab25bf..9f540d634d06 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -42,6 +42,8 @@ import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.TestTimelineServerView; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; @@ -57,6 +59,8 @@ import org.apache.druid.sql.calcite.planner.PlannerFactory; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.run.SqlEngine; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache; +import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.schema.DruidSchema; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.schema.DruidSchemaManager; @@ -68,7 +72,7 @@ import org.apache.druid.sql.calcite.schema.NamedSystemSchema; import org.apache.druid.sql.calcite.schema.NamedViewSchema; import org.apache.druid.sql.calcite.schema.NoopDruidSchemaManager; -import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.sql.calcite.schema.PhysicalDatasourceMetadataBuilder; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.sql.calcite.schema.ViewSchema; import org.apache.druid.sql.calcite.view.ViewManager; @@ -143,11 +147,10 @@ public static DruidSchemaCatalog createMockRootSchema( injector, conglomerate, walker, - plannerConfig, druidSchemaManager ); SystemSchema systemSchema = - CalciteTests.createMockSystemSchema(druidSchema, walker, authorizerMapper); + CalciteTests.createMockSystemSchema(createMockQueryLifecycleFactory(walker, conglomerate), walker, authorizerMapper); LookupSchema lookupSchema = createMockLookupSchema(injector); ViewSchema viewSchema = viewManager != null ? new ViewSchema(viewManager) : null; @@ -186,7 +189,7 @@ public static DruidSchemaCatalog createMockRootSchema( public static DruidSchemaCatalog createMockRootSchema( final Injector injector, - final QueryRunnerFactoryConglomerate conglomerate, + final QueryRunnerFactoryConglomerate conglomerate, final SpecificSegmentsQuerySegmentWalker walker, final PlannerConfig plannerConfig, final AuthorizerMapper authorizerMapper @@ -207,26 +210,27 @@ private static DruidSchema createMockSchema( final Injector injector, final QueryRunnerFactoryConglomerate conglomerate, final SpecificSegmentsQuerySegmentWalker walker, - final PlannerConfig plannerConfig, final DruidSchemaManager druidSchemaManager ) { - final SegmentMetadataCache cache = new SegmentMetadataCache( + final BrokerSegmentMetadataCache cache = new BrokerSegmentMetadataCache( createMockQueryLifecycleFactory(walker, conglomerate), - new TestServerInventoryView(walker.getSegments()), - new SegmentManager(EasyMock.createMock(SegmentLoader.class)) - { - @Override - public Set getDataSourceNames() - { - return ImmutableSet.of(CalciteTests.BROADCAST_DATASOURCE); - } - }, - createDefaultJoinableFactory(injector), - SegmentMetadataCacheConfig.create(), + new TestTimelineServerView(walker.getSegments()), + BrokerSegmentMetadataCacheConfig.create(), CalciteTests.TEST_AUTHENTICATOR_ESCALATOR, new InternalQueryConfig(), - new NoopServiceEmitter() + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder( + createDefaultJoinableFactory(injector), + new SegmentManager(EasyMock.createMock(SegmentLoader.class)) + { + @Override + public Set getDataSourceNames() + { + return ImmutableSet.of(CalciteTests.BROADCAST_DATASOURCE); + } + }), + null ); try { @@ -274,5 +278,5 @@ public static DruidOperatorTable createOperatorTable(final Injector injector) public static LookupSchema createMockLookupSchema(final Injector injector) { return new LookupSchema(injector.getInstance(LookupExtractorFactoryContainerProvider.class)); - } **/ + } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 3467d71bb627..7e012b435811 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -40,6 +40,7 @@ import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.segment.join.JoinableFactoryWrapper; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java index b2f93340dbf7..edfa2fc2d334 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java @@ -61,6 +61,7 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.join.table.IndexedTableJoinable; import org.apache.druid.segment.join.table.RowBasedIndexedTable; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QueryStackTests; diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index 98f1f7b1cab5..eefbe389e62d 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -103,7 +103,7 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.junit.After; From 95389b177f5dd3da2c3686d17213e118ce82a489 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 8 Sep 2023 21:49:21 +0530 Subject: [PATCH 10/36] Fix unit tests and add test for getAllUsedSegments --- .../timeline/SegmentStatusInCluster.java | 22 +-- .../druid/client/CoordinatorServerView.java | 19 ++ .../druid/client/CoordinatorTimeline.java | 21 +- .../client/coordinator/CoordinatorClient.java | 4 +- .../coordinator/CoordinatorClientImpl.java | 8 +- ...Schema.java => DataSourceInformation.java} | 6 +- .../metadata/SegmentMetadataCache.java | 16 +- .../druid/server/http/MetadataResource.java | 48 +++-- .../coordinator/NoopCoordinatorClient.java | 4 +- .../SegmentDataCacheConcurrencyTest.java | 4 +- .../SegmentMetadataCacheConfigTest.java | 2 +- .../metadata/SegmentMetadataCacheTest.java | 16 +- .../server/http/MetadataResourceTest.java | 133 +++++++++++-- .../schema/BrokerSegmentMetadataCache.java | 15 +- .../schema/BrokerSegmentMetadataView.java | 45 +++-- .../PhysicalDatasourceMetadataBuilder.java | 5 +- .../sql/calcite/schema/SystemSchema.java | 182 ++++++------------ .../org/apache/druid/sql/guice/SqlModule.java | 1 - .../schema/DruidCalciteSchemaModuleTest.java | 3 + .../sql/calcite/schema/SystemSchemaTest.java | 99 ++++++++-- .../druid/sql/calcite/util/CalciteTests.java | 8 +- .../apache/druid/sql/guice/SqlModuleTest.java | 3 + 22 files changed, 420 insertions(+), 244 deletions(-) rename server/src/main/java/org/apache/druid/segment/metadata/{DataSourceSchema.java => DataSourceInformation.java} (91%) diff --git a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java index a440a36923e6..b550a9c8d421 100644 --- a/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java +++ b/processing/src/main/java/org/apache/druid/timeline/SegmentStatusInCluster.java @@ -54,27 +54,25 @@ public class SegmentStatusInCluster implements Comparable getSegmentLoadInfos(); - - } diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index 66b739cf51cb..b8f384ae1ba7 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -22,7 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; -import org.apache.druid.segment.metadata.DataSourceSchema; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; @@ -49,7 +49,7 @@ public interface CoordinatorClient /** * Fetches schema for the given dataSources. */ - ListenableFuture> fetchDataSourceSchema(Set datasources); + ListenableFuture> fetchDataSourceInformation(Set datasources); /** * Returns a new instance backed by a ServiceClient which follows the provided retryPolicy diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index 24e6968ad6ab..f11c3de2eaec 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -30,7 +30,7 @@ import org.apache.druid.rpc.RequestBuilder; import org.apache.druid.rpc.ServiceClient; import org.apache.druid.rpc.ServiceRetryPolicy; -import org.apache.druid.segment.metadata.DataSourceSchema; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.timeline.DataSegment; import org.jboss.netty.handler.codec.http.HttpMethod; import org.joda.time.Interval; @@ -110,9 +110,9 @@ public ListenableFuture> fetchUsedSegments(String dataSource, } @Override - public ListenableFuture> fetchDataSourceSchema(Set dataSources) + public ListenableFuture> fetchDataSourceInformation(Set dataSources) { - final String path = "/druid/coordinator/v1/metadata/dataSourceSchema"; + final String path = "/druid/coordinator/v1/metadata/dataSourceInformation"; if (null == dataSources) { dataSources = new HashSet<>(); @@ -123,7 +123,7 @@ public ListenableFuture> fetchDataSourceSchema(Set JacksonUtils.readValue(jsonMapper, holder.getContent(), new TypeReference>() {}) + holder -> JacksonUtils.readValue(jsonMapper, holder.getContent(), new TypeReference>() {}) ); } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java similarity index 91% rename from server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java rename to server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index 0528cfe3b1f8..6501704bddac 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceSchema.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -9,13 +9,13 @@ /** * Encapsulates schema information of a dataSource. */ -public class DataSourceSchema +public class DataSourceInformation { private final String datasource; private final RowSignature rowSignature; @JsonCreator - public DataSourceSchema( + public DataSourceInformation( @JsonProperty("datasource") String datasource, @JsonProperty("rowSignature") RowSignature rowSignature) { @@ -44,7 +44,7 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } - DataSourceSchema that = (DataSourceSchema) o; + DataSourceInformation that = (DataSourceInformation) o; return Objects.equals(datasource, that.datasource) && Objects.equals( rowSignature, that.rowSignature diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 56328fbc3dec..5077ed046577 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -124,7 +124,7 @@ public class SegmentMetadataCache * Map of DataSource -> DruidTable. * This map can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final ConcurrentMap tables = new ConcurrentHashMap<>(); /** * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. @@ -201,7 +201,7 @@ public class SegmentMetadataCache * Currently, there are 2 threads that can access these variables. * * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. - * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceSchema} if necessary + * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceInformation} if necessary * based on the information collected via timeline callbacks. */ protected final Object lock = new Object(); @@ -428,13 +428,13 @@ public void refresh(final Set segmentsToRefresh, final Set da public void rebuildDatasource(String dataSource) { - final DataSourceSchema druidTable = buildDruidTable(dataSource); + final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } - final DataSourceSchema oldTable = tables.put(dataSource, druidTable); + final DataSourceInformation oldTable = tables.put(dataSource, druidTable); if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { @@ -454,12 +454,12 @@ public void awaitInitialization() throws InterruptedException initialized.await(); } - public DataSourceSchema getDatasource(String name) + public DataSourceInformation getDatasource(String name) { return tables.get(name); } - public Map getDataSourceSchemaMap() + public Map getDataSourceSchemaMap() { return ImmutableMap.copyOf(tables); } @@ -818,7 +818,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin @VisibleForTesting @Nullable - public DataSourceSchema buildDruidTable(final String dataSource) + public DataSourceInformation buildDruidTable(final String dataSource) { ConcurrentSkipListMap segmentsMap = segmentMetadataInfo.get(dataSource); @@ -846,7 +846,7 @@ public DataSourceSchema buildDruidTable(final String dataSource) final RowSignature.Builder builder = RowSignature.builder(); columnTypes.forEach(builder::add); - return new DataSourceSchema(dataSource, builder.build()); + return new DataSourceInformation(dataSource, builder.build()); } public Map getSegmentMetadataSnapshot() diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 1de89a11de73..92e5dd03ba40 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -31,7 +31,7 @@ import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; -import org.apache.druid.segment.metadata.DataSourceSchema; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.JettyUtils; import org.apache.druid.server.coordinator.DruidCoordinator; @@ -201,6 +201,9 @@ private Response getAllUsedSegmentsWithAdditionalDetails( .map(segment -> { // The replication factor for unloaded segments is 0 as they will be unloaded soon boolean isOvershadowed = overshadowedSegments.contains(segment); + Integer replicationFactor = isOvershadowed ? (Integer) 0 + : coordinator.getReplicationFactor(segment.getId()); + Long numRows = null; if (null != segmentMetadataCache) { AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata( @@ -211,29 +214,34 @@ private Response getAllUsedSegmentsWithAdditionalDetails( numRows = availableSegmentMetadata.getNumRows(); } } - Integer replicationFactor = isOvershadowed ? (Integer) 0 - : coordinator.getReplicationFactor(segment.getId()); segmentAlreadySeen.add(segment.getId()); - return new SegmentStatusInCluster(segment, isOvershadowed, replicationFactor, numRows, true); + return new SegmentStatusInCluster( + segment, + isOvershadowed, + replicationFactor, + numRows, + // published segment can't be realtime + false); }); Stream finalSegments = segmentStatus; - if (null != includeRealtimeSegments && segmentMetadataCache != null) { - log.info("printing the content in smc cache %s", segmentMetadataCache.getSegmentMetadataSnapshot().values()); + if (null != includeRealtimeSegments && null != segmentMetadataCache) { final Stream realtimeSegmentStatus = segmentMetadataCache .getSegmentMetadataSnapshot() .values() .stream() - .filter(availableSegmentMetadata -> !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment() - .getId())) - .map(availableSegmentMetadata -> new SegmentStatusInCluster( - availableSegmentMetadata.getSegment(), - false, - (int) availableSegmentMetadata.getNumReplicas(), - availableSegmentMetadata.getNumRows(), - availableSegmentMetadata.isRealtime() != 0 - )); + .filter(availableSegmentMetadata -> + !segmentAlreadySeen.contains(availableSegmentMetadata.getSegment().getId())) + .map(availableSegmentMetadata -> + new SegmentStatusInCluster( + availableSegmentMetadata.getSegment(), + false, + // replication factor is null for unpublished segments + null, + availableSegmentMetadata.getNumRows(), + availableSegmentMetadata.isRealtime() != 0 + )); finalSegments = Stream.concat(segmentStatus, realtimeSegmentStatus); } @@ -351,21 +359,21 @@ public Response getUsedSegment( * Return schema for the given dataSources. */ @POST - @Path("/dataSourceSchema") + @Path("/dataSourceInformation") @Produces(MediaType.APPLICATION_JSON) - public Response getDataSourceSchema( + public Response getDataSourceInformation( List dataSources ) { if (null == segmentMetadataCache) { return Response.status(Response.Status.NOT_FOUND).build(); } - Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceSchemaMap(); + Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceSchemaMap(); - List results = new ArrayList<>(); + List results = new ArrayList<>(); List dataSourcesToRetain = (null == dataSources) ? new ArrayList<>(dataSourceSchemaMap.keySet()) : dataSources; - for (Map.Entry entry : dataSourceSchemaMap.entrySet()) { + for (Map.Entry entry : dataSourceSchemaMap.entrySet()) { if (dataSourcesToRetain.contains(entry.getKey())) { results.add(entry.getValue()); } diff --git a/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java b/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java index b2cde828092e..bbceeb4523e1 100644 --- a/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java +++ b/server/src/test/java/org/apache/druid/client/coordinator/NoopCoordinatorClient.java @@ -22,7 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.ServiceRetryPolicy; -import org.apache.druid.segment.metadata.DataSourceSchema; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.timeline.DataSegment; import org.joda.time.Interval; @@ -50,7 +50,7 @@ public ListenableFuture> fetchUsedSegments(String dataSource, } @Override - public ListenableFuture> fetchDataSourceSchema(Set datasources) + public ListenableFuture> fetchDataSourceInformation(Set datasources) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index 92ad857476e1..60421be19810 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -138,7 +138,7 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV ) { @Override - public DataSourceSchema buildDruidTable(final String dataSource) + public DataSourceInformation buildDruidTable(final String dataSource) { doInLock(() -> { try { @@ -246,7 +246,7 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() ) { @Override - public DataSourceSchema buildDruidTable(final String dataSource) + public DataSourceInformation buildDruidTable(final String dataSource) { doInLock(() -> { try { diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java index 16567719dffc..cf0511dd6f8a 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java @@ -51,7 +51,7 @@ public void testDefaultConfig() final Properties properties = new Properties(); provider.inject(properties, injector.getInstance(JsonConfigurator.class)); final SegmentMetadataCacheConfig config = provider.get(); - Assert.assertTrue(config.isAwaitInitializationOnStart()); + Assert.assertFalse(config.isAwaitInitializationOnStart()); Assert.assertEquals(Period.minutes(1), config.getMetadataRefreshPeriod()); Assert.assertEquals(new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index dc18cfb79741..aa9df38b9b71 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -288,9 +288,9 @@ public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheCon ) { @Override - public DataSourceSchema buildDruidTable(String dataSource) + public DataSourceInformation buildDruidTable(String dataSource) { - DataSourceSchema table = super.buildDruidTable(dataSource); + DataSourceInformation table = super.buildDruidTable(dataSource); buildTableLatch.countDown(); return table; } @@ -371,14 +371,14 @@ public void testSchemaInit() throws InterruptedException public void testGetTableMapFoo() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DataSourceSchema fooDs = schema.getDatasource("foo"); + final DataSourceInformation fooDs = schema.getDatasource("foo"); } @Test public void testGetTableMapFoo2() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DataSourceSchema fooDs = schema.getDatasource("foo2"); + final DataSourceInformation fooDs = schema.getDatasource("foo2"); } @Test @@ -395,7 +395,7 @@ public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePoli } } ); - final DataSourceSchema fooDs = schema.getDatasource(SOME_DATASOURCE); + final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); } @@ -405,7 +405,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DataSourceSchema fooDs = schema.getDatasource(SOME_DATASOURCE); + final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); } @@ -1002,7 +1002,7 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte { SegmentMetadataCache schema3 = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DataSourceSchema fooTable = schema3.getDatasource("foo"); + DataSourceInformation fooTable = schema3.getDatasource("foo"); Assert.assertNotNull(fooTable); markDataSourceLatch = new CountDownLatch(1); @@ -1052,7 +1052,7 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw { SegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DataSourceSchema fooTable = schema.getDatasource("foo"); + DataSourceInformation fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); markDataSourceLatch = new CountDownLatch(1); diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index c0978e80c367..589d2a3ae52f 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -19,21 +19,26 @@ package org.apache.druid.server.http; +import com.google.api.client.util.Sets; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.druid.client.DataSourcesSnapshot; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.metadata.SegmentsMetadataManager; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.coordinator.CreateDataSegments; import org.apache.druid.server.coordinator.DruidCoordinator; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.junit.Assert; import org.junit.Before; @@ -42,22 +47,29 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.Response; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; public class MetadataResourceTest { private static final String DATASOURCE1 = "datasource1"; - private MetadataResource metadataResource; - private HttpServletRequest request; - private final DataSegment[] segments = CreateDataSegments.ofDatasource(DATASOURCE1) .forIntervals(3, Granularities.DAY) .withNumPartitions(2) .eachOfSizeInMb(500) .toArray(new DataSegment[0]); - + private HttpServletRequest request; + private SegmentsMetadataManager segmentsMetadataManager; + private IndexerMetadataStorageCoordinator storageCoordinator; + private DruidCoordinator coordinator; + + + private MetadataResource metadataResource; + @Before public void setUp() { @@ -65,7 +77,7 @@ public void setUp() Mockito.doReturn(Mockito.mock(AuthenticationResult.class)) .when(request).getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT); - SegmentsMetadataManager segmentsMetadataManager = Mockito.mock(SegmentsMetadataManager.class); + segmentsMetadataManager = Mockito.mock(SegmentsMetadataManager.class); ImmutableDruidDataSource druidDataSource1 = new ImmutableDruidDataSource( DATASOURCE1, ImmutableMap.of(), @@ -81,7 +93,7 @@ public void setUp() .when(segmentsMetadataManager) .getImmutableDataSourceWithUsedSegments(DATASOURCE1); - DruidCoordinator coordinator = Mockito.mock(DruidCoordinator.class); + coordinator = Mockito.mock(DruidCoordinator.class); Mockito.doReturn(2).when(coordinator).getReplicationFactor(segments[0].getId()); Mockito.doReturn(null).when(coordinator).getReplicationFactor(segments[1].getId()); Mockito.doReturn(1).when(coordinator).getReplicationFactor(segments[2].getId()); @@ -89,7 +101,7 @@ public void setUp() Mockito.doReturn(ImmutableSet.of(segments[3])) .when(dataSourcesSnapshot).getOvershadowedSegments(); - IndexerMetadataStorageCoordinator storageCoordinator = Mockito.mock(IndexerMetadataStorageCoordinator.class); + storageCoordinator = Mockito.mock(IndexerMetadataStorageCoordinator.class); Mockito.doReturn(segments[4]) .when(storageCoordinator) .retrieveUsedSegmentForId(segments[4].getId().toString()); @@ -113,11 +125,110 @@ public void testGetAllSegmentsWithOvershadowedStatus() final List resultList = extractSegmentStatusList(response); Assert.assertEquals(resultList.size(), 4); - Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 5L, true), resultList.get(0)); - Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 5L, true), resultList.get(1)); - Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, 5L, true), resultList.get(2)); + Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, null, false), resultList.get(0)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, null, false), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, null, false), resultList.get(2)); + // Replication factor should be 0 as the segment is overshadowed + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); + } + + @Test + public void testGetAllSegmentsIncludingRealtime() + { + SegmentMetadataCache segmentMetadataCache = Mockito.mock(SegmentMetadataCache.class); + + String dataSource2 = "datasource2"; + + DataSegment[] realTimeSegments = + CreateDataSegments.ofDatasource(dataSource2) + .forIntervals(3, Granularities.DAY) + .withNumPartitions(2) + .eachOfSizeInMb(500) + .toArray(new DataSegment[0]); + + Mockito.doReturn(null).when(coordinator).getReplicationFactor(realTimeSegments[0].getId()); + Mockito.doReturn(null).when(coordinator).getReplicationFactor(realTimeSegments[1].getId()); + Map availableSegments = new HashMap<>(); + availableSegments.put( + segments[0].getId(), + AvailableSegmentMetadata.builder( + segments[0], + 0L, + Sets.newHashSet(), + null, + 20L + ).build() + ); + availableSegments.put( + segments[1].getId(), + AvailableSegmentMetadata.builder( + segments[1], + 0L, + Sets.newHashSet(), + null, + 30L + ).build() + ); + availableSegments.put( + segments[1].getId(), + AvailableSegmentMetadata.builder( + segments[1], + 0L, + Sets.newHashSet(), + null, + 30L + ).build() + ); + availableSegments.put( + realTimeSegments[0].getId(), + AvailableSegmentMetadata.builder( + realTimeSegments[0], + 1L, + Sets.newHashSet(), + null, + 10L + ).build() + ); + availableSegments.put( + realTimeSegments[1].getId(), + AvailableSegmentMetadata.builder( + realTimeSegments[1], + 1L, + Sets.newHashSet(), + null, + 40L + ).build() + ); + + Mockito.doReturn(availableSegments).when(segmentMetadataCache).getSegmentMetadataSnapshot(); + + Mockito.doReturn(availableSegments.get(segments[0].getId())) + .when(segmentMetadataCache) + .getAvailableSegmentMetadata(DATASOURCE1, segments[0].getId()); + + Mockito.doReturn(availableSegments.get(segments[1].getId())) + .when(segmentMetadataCache) + .getAvailableSegmentMetadata(DATASOURCE1, segments[1].getId()); + + metadataResource = new MetadataResource( + segmentsMetadataManager, + storageCoordinator, + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + coordinator, + segmentMetadataCache + ); + + Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus", "includeRealtimeSegments"); + + final List resultList = extractSegmentStatusList(response); + Assert.assertEquals(resultList.size(), 6); + Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 20L, false), resultList.get(0)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 30L, false), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, null, false), resultList.get(2)); // Replication factor should be 0 as the segment is overshadowed - Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, 5L, true), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[0], false, null, 10L, true), resultList.get(4)); + Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[1], false, null, 40L, true), resultList.get(5)); } @Test diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 9fa0f3938b81..0a0428ad1d50 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -28,9 +28,8 @@ import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.segment.metadata.DataSourceSchema; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.table.DatasourceTable; @@ -92,13 +91,10 @@ public void refresh(final Set segmentsToRefresh, final Set da // Fetch dataSource schema from the Coordinator try { - FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceSchema(dataSourcesToQuery), true) + FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) .forEach(item -> polledDataSourceSchema.put( item.getDatasource(), - physicalDatasourceMetadataBuilder.build( - item.getDatasource(), - item.getRowSignature() - ) + physicalDatasourceMetadataBuilder.build(item) )); } catch (Exception e) { log.error("Exception querying coordinator for schema"); @@ -140,14 +136,13 @@ public void refresh(final Set segmentsToRefresh, final Set da @Override public void rebuildDatasource(String dataSource) { - final DataSourceSchema druidTable = buildDruidTable(dataSource); + final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } - final DatasourceTable.PhysicalDatasourceMetadata physicalDatasourceMetadata = - physicalDatasourceMetadataBuilder.build(dataSource, druidTable.getRowSignature()); + final DatasourceTable.PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); final DatasourceTable.PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java index 9ef381b10b3f..38b570b779d1 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java @@ -28,11 +28,9 @@ import com.google.common.util.concurrent.Uninterruptibles; import com.google.inject.Inject; import org.apache.druid.client.BrokerSegmentWatcherConfig; -import org.apache.druid.client.BrokerServerView; import org.apache.druid.client.DataSegmentInterner; import org.apache.druid.client.JsonParserIterator; import org.apache.druid.client.coordinator.Coordinator; -import org.apache.druid.client.selector.ServerSelector; import org.apache.druid.concurrent.LifecycleLock; import org.apache.druid.discovery.DruidLeaderClient; import org.apache.druid.guice.ManageLifecycle; @@ -74,11 +72,8 @@ public class BrokerSegmentMetadataView private final BrokerSegmentWatcherConfig segmentWatcherConfig; private final DruidLeaderClient druidLeaderClient; private final ObjectMapper objectMapper; - - private final boolean isMetadataSegmentCacheEnabled; - /** * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and * sys.segments queries return the segments in sorted order based on segmentId. @@ -161,14 +156,16 @@ public void stop() protected Iterator getSegmentTableView() { + // set of published and realtime segments final ImmutableSortedSet segments = getSegmentMetadata(); + + // set of available segments and metadata final Map availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); final List segmentsTableViews = new ArrayList<>(); Set seenSegments = new HashSet<>(); - for (SegmentStatusInCluster segmentStatusInCluster : segments) - { + for (SegmentStatusInCluster segmentStatusInCluster : segments) { DataSegment segment = segmentStatusInCluster.getDataSegment(); SegmentId segmentId = segment.getId(); AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataMap.get(segmentId); @@ -188,6 +185,12 @@ protected Iterator getSegmentTableView() long isRealtime = Boolean.TRUE.equals(segmentStatusInCluster.isRealtime()) ? 1 : 0; + // set of segments returned from coordinator include published and realtime segments + // so realtime segments are not published and vice versa + boolean isPublished = !segmentStatusInCluster.isRealtime(); + + // is_active is true for published segments that are not overshadowed + boolean isActive = isPublished && !segmentStatusInCluster.isOvershadowed(); SegmentTableView segmentTableView = new SegmentTableView( segment, @@ -195,28 +198,48 @@ protected Iterator getSegmentTableView() isRealtime, numReplicas, numRows, + // If the segment is unpublished, we won't have this information yet. + // If the value is null, the load rules might have not evaluated yet, and we don't know the replication factor. + // This should be automatically updated in the next refesh with Coordinator. segmentStatusInCluster.getReplicationFactor(), - segmentStatusInCluster.isOvershadowed() + // If the segment is unpublished this value would be false + segmentStatusInCluster.isOvershadowed(), + isPublished, + isActive ); + seenSegments.add(segmentId); segmentsTableViews.add((segmentTableView)); } - for (Map.Entry availableSegmentMetadataEntry : availableSegmentMetadataMap.entrySet()) - { + for (Map.Entry availableSegmentMetadataEntry : availableSegmentMetadataMap.entrySet()) { if (seenSegments.contains(availableSegmentMetadataEntry.getKey())) { continue; } AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataEntry.getValue(); + + // since all published segments would have been covered in the previous loop + // the curent set of segments must be unpublished + boolean isPublished = false; + + // is_active is true for unpublished segments iff they are realtime + boolean isActive = availableSegmentMetadata.isRealtime() == 1L; + SegmentTableView segmentTableView = new SegmentTableView( availableSegmentMetadata.getSegment(), + // segments announced by historicals or realtime tasks are assumed to be available 1L, availableSegmentMetadata.isRealtime(), availableSegmentMetadata.getNumReplicas(), availableSegmentMetadata.getNumRows(), + // If the segment is unpublished, we won't have this information yet. null, - false + // there is an assumption here that unpublished segments are never overshadowed + false, + isPublished, + isActive ); + segmentsTableViews.add(segmentTableView); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index 5dfbfb664440..5b344621f1f8 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -24,6 +24,7 @@ import org.apache.druid.query.TableDataSource; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.SegmentManager; import org.apache.druid.sql.calcite.table.DatasourceTable; @@ -39,8 +40,10 @@ public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, Segmen this.segmentManager = segmentManager; } - DatasourceTable.PhysicalDatasourceMetadata build(String dataSource, RowSignature rowSignature) + DatasourceTable.PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) { + final String dataSource = dataSourceInformation.getDatasource(); + final RowSignature rowSignature = dataSourceInformation.getRowSignature(); final TableDataSource tableDataSource; // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 28e322a3ddca..afa0986467d5 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -27,10 +27,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSortedSet; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import com.google.common.net.HostAndPort; import com.google.common.util.concurrent.Futures; import com.google.inject.Inject; @@ -72,7 +68,6 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.DruidNode; -import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; import org.apache.druid.server.security.AuthenticationResult; @@ -85,7 +80,6 @@ import org.apache.druid.sql.calcite.table.RowSignatures; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.SegmentStatusInCluster; import org.jboss.netty.handler.codec.http.HttpMethod; import javax.annotation.Nullable; @@ -100,7 +94,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; public class SystemSchema extends AbstractSchema @@ -122,20 +115,6 @@ public class SystemSchema extends AbstractSchema segment.getDataSource()) ); - private static final long REPLICATION_FACTOR_UNKNOWN = -1L; - - /** - * Booleans constants represented as long type, - * where 1 = true and 0 = false to make it easy to count number of segments - * which are published, available etc. - */ - private static final long IS_ACTIVE_FALSE = 0L; - private static final long IS_ACTIVE_TRUE = 1L; - private static final long IS_PUBLISHED_FALSE = 0L; - private static final long IS_PUBLISHED_TRUE = 1L; - private static final long IS_OVERSHADOWED_FALSE = 0L; - private static final long IS_OVERSHADOWED_TRUE = 1L; - static final RowSignature SEGMENTS_SIGNATURE = RowSignature .builder() .add("segment_id", ColumnType.STRING) @@ -291,22 +270,18 @@ public Enumerable scan(DataContext root) (long) segment.getShardSpec().getPartitionNum(), val.getNumReplicas(), val.getNumRows(), - //is_active is true for published segments that are not overshadowed - val.isOvershadowed() ? IS_ACTIVE_FALSE : IS_ACTIVE_TRUE, - //is_published is true for published segments - (val.getIsRealtime() == 0) ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE, - val.getIsAvailable(), - val.getIsRealtime(), - val.isOvershadowed() ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE, + val.isActive(), + val.isPublished(), + val.isAvailable(), + val.isRealtime(), + val.isOvershadowed(), segment.getShardSpec() == null ? null : jsonMapper.writeValueAsString(segment.getShardSpec()), segment.getDimensions() == null ? null : jsonMapper.writeValueAsString(segment.getDimensions()), segment.getMetrics() == null ? null : jsonMapper.writeValueAsString(segment.getMetrics()), segment.getLastCompactionState() == null ? null : jsonMapper.writeValueAsString(segment.getLastCompactionState()), - // If the value is null, the load rules might have not evaluated yet, and we don't know the replication factor. - // This should be automatically updated in the next refesh with Coordinator. - val.getReplicationFactor() == null ? REPLICATION_FACTOR_UNKNOWN : (long) val.getReplicationFactor() + val.getReplicationFactor() }; } catch (JsonProcessingException e) { @@ -366,31 +341,50 @@ private Iterator> getAuthorizedAvaila protected static class SegmentTableView { + /** + * Booleans constants represented as long type, + * where 1 = true and 0 = false to make it easy to count number of segments + * which are published, available etc. + */ + private static final long IS_ACTIVE_FALSE = 0L; + private static final long IS_ACTIVE_TRUE = 1L; + private static final long IS_PUBLISHED_FALSE = 0L; + private static final long IS_PUBLISHED_TRUE = 1L; + private static final long IS_OVERSHADOWED_FALSE = 0L; + private static final long IS_OVERSHADOWED_TRUE = 1L; + private static final long REPLICATION_FACTOR_UNKNOWN = -1L; + private final DataSegment segment; - private final long isAvailable; - private final long isRealtime; + private final long available; + private final long realtime; private final long numReplicas; private final long numRows; - private final Integer replicationFactor; - private final boolean isOvershadowed; + private final long replicationFactor; + private final long overshadowed; + private final long published; + private final long active; public SegmentTableView( DataSegment segment, long isAvailable, - long isRealtime, + long realtime, long numReplicas, long numRows, Integer replicationFactor, - boolean isOvershadowed + boolean overshadowed, + boolean isPublished, + boolean active ) { this.segment = segment; - this.isAvailable = isAvailable; - this.isRealtime = isRealtime; + this.available = isAvailable; + this.realtime = realtime; this.numReplicas = numReplicas; this.numRows = numRows; - this.replicationFactor = replicationFactor; - this.isOvershadowed = isOvershadowed; + this.replicationFactor = (null == replicationFactor) ? REPLICATION_FACTOR_UNKNOWN : (long) replicationFactor; + this.overshadowed = overshadowed ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE; + this.published = isPublished ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE; + this.active = active ? IS_ACTIVE_TRUE : IS_ACTIVE_FALSE; } public DataSegment getSegment() @@ -398,14 +392,14 @@ public DataSegment getSegment() return segment; } - public long getIsAvailable() + public long isAvailable() { - return isAvailable; + return available; } - public long getIsRealtime() + public long isRealtime() { - return isRealtime; + return realtime; } public long getNumReplicas() @@ -418,14 +412,24 @@ public long getNumRows() return numRows; } - public Integer getReplicationFactor() + public long getReplicationFactor() { return replicationFactor; } - public boolean isOvershadowed() + public long isOvershadowed() { - return isOvershadowed; + return overshadowed; + } + + public long isPublished() + { + return published; + } + + public long isActive() + { + return active; } @Override @@ -433,90 +437,16 @@ public String toString() { return "SegmentTableView{" + "segmentId=" + segment.getId() + - ", isAvailable=" + isAvailable + - ", isRealtime=" + isRealtime + + ", isAvailable=" + available + + ", isRealtime=" + realtime + ", numReplicas=" + numReplicas + ", numRows=" + numRows + ", replicationFactor=" + replicationFactor + - ", isOvershadowed=" + isOvershadowed + + ", isOvershadowed=" + overshadowed + + ", isPublished=" + published + '}'; } } - - protected static class SegmentTableView1 - { - // pass the stitched - private final Map availableSegmentMetadata; - private final Iterator publishedSegments; // coordinator - - private final int totalSegmentsCount; - - public SegmentTableView1( - Map availableSegmentMetadata, - ImmutableSortedSet publishedSegments - ) - { - this.availableSegmentMetadata = availableSegmentMetadata; - this.publishedSegments = publishedSegments.iterator(); - this.totalSegmentsCount = availableSegmentMetadata.size(); - } - - public Map getAvailableSegmentMetadata() - { - return availableSegmentMetadata; - } - - public Iterator getPublishedSegments() - { - return publishedSegments; - } - - public int totalSegmentCount() - { - return totalSegmentsCount; - } - } - - protected static class PartialSegmentData - { - private final long isAvailable; - private final long isRealtime; - private final long numReplicas; - private final long numRows; - - public PartialSegmentData( - final long isAvailable, - final long isRealtime, - final long numReplicas, - final long numRows - ) - { - this.isAvailable = isAvailable; - this.isRealtime = isRealtime; - this.numReplicas = numReplicas; - this.numRows = numRows; - } - - public long isAvailable() - { - return isAvailable; - } - - public long isRealtime() - { - return isRealtime; - } - - public long getNumReplicas() - { - return numReplicas; - } - - public long getNumRows() - { - return numRows; - } - } } /** diff --git a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java index 56d0d2d5d41f..63ef71585dd9 100644 --- a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java +++ b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java @@ -62,7 +62,6 @@ public class SqlModule implements Module public static final String PROPERTY_SQL_VIEW_MANAGER_TYPE = "druid.sql.viewmanager.type"; public static final String PROPERTY_SQL_SCHEMA_MANAGER_TYPE = "druid.sql.schemamanager.type"; public static final String PROPERTY_SQL_APPROX_COUNT_DISTINCT_CHOICE = "druid.sql.approxCountDistinct.function"; - private Properties props; @Inject diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModuleTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModuleTest.java index 1d646214ec74..0b5fb609fa22 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModuleTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModuleTest.java @@ -31,6 +31,8 @@ import org.apache.druid.client.FilteredServerInventoryView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.client.coordinator.CoordinatorClient; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.client.indexing.IndexingService; import org.apache.druid.client.indexing.NoopOverlordClient; import org.apache.druid.discovery.DruidLeaderClient; @@ -132,6 +134,7 @@ public void setUp() binder.bind(CatalogResolver.class).toInstance(CatalogResolver.NULL_RESOLVER); binder.bind(ServiceEmitter.class).toInstance(new ServiceEmitter("", "", null)); binder.bind(OverlordClient.class).to(NoopOverlordClient.class); + binder.bind(CoordinatorClient.class).to(NoopCoordinatorClient.class); }, new LifecycleModule(), target); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index b17eb7713688..a159dcc24e5c 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -19,6 +19,7 @@ package org.apache.druid.sql.calcite.schema; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; @@ -36,6 +37,7 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; @@ -62,6 +64,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.http.client.Request; +import org.apache.druid.java.util.http.client.response.FullResponseHolder; import org.apache.druid.java.util.http.client.response.HttpResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; @@ -106,9 +109,14 @@ import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.SegmentStatusInCluster; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.easymock.EasyMock; +import org.jboss.netty.handler.codec.http.DefaultHttpResponse; +import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponse; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.jboss.netty.handler.codec.http.HttpVersion; import org.joda.time.DateTime; import org.junit.AfterClass; import org.junit.Assert; @@ -122,6 +130,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -153,7 +162,6 @@ public class SystemSchemaTest extends CalciteTestBase ); private SystemSchema schema; - private SpecificSegmentsQuerySegmentWalker walker; private DruidLeaderClient client; private DruidLeaderClient coordinatorClient; private OverlordClient overlordClient; @@ -162,7 +170,6 @@ public class SystemSchemaTest extends CalciteTestBase private StringFullResponseHolder responseHolder; private BytesAccumulatingResponseHandler responseHandler; private Request request; - private DruidSchema druidSchema; private AuthorizerMapper authMapper; private static QueryRunnerFactoryConglomerate conglomerate; private static Closer resourceCloser; @@ -247,7 +254,7 @@ public void setUp() throws Exception .rows(ROWS3) .buildMMappedIndex(); - walker = new SpecificSegmentsQuerySegmentWalker(conglomerate) + SpecificSegmentsQuerySegmentWalker walker = new SpecificSegmentsQuerySegmentWalker(conglomerate) .add(segment1, index1) .add(segment2, index2) .add(segment3, index3); @@ -265,10 +272,69 @@ public void setUp() throws Exception ), new NoopCoordinatorClient() ); + cache.start(); cache.awaitInitialization(); - druidSchema = new DruidSchema(cache, null); - metadataView = EasyMock.createMock(BrokerSegmentMetadataView.class); + + DruidNode coordinatorNode = new DruidNode("test-coordinator", "localhost", false, 8081, null, true, false); + DruidLeaderClient druidLeaderClient = new DruidLeaderClient( + new CalciteTests.FakeHttpClient(), + new CalciteTests.FakeDruidNodeDiscoveryProvider( + ImmutableMap.of( + NodeRole.COORDINATOR, + new CalciteTests.FakeDruidNodeDiscovery(ImmutableMap.of(NodeRole.COORDINATOR, coordinatorNode)) + ) + ), + NodeRole.COORDINATOR, + "/simple/leader" + ) + { + @Override + public String findCurrentLeader() + { + return coordinatorNode.getHostAndPortToUse(); + } + + @Override + public Request makeRequest(HttpMethod httpMethod, String urlPath) throws IOException + { + return new Request(httpMethod, new URL(StringUtils.format("http://%s%s", coordinatorNode.getHostAndPortToUse(), urlPath))); + } + + @Override + public > H go(Request request, HttpResponseHandler responseHandler) + throws IOException + { + List segmentStatusInClusterList = new ArrayList<>(Arrays.asList( + new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 0L, false), + new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 0L, false), + new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 0L, false), + new SegmentStatusInCluster(segment1, true, 2, 3L, false), + new SegmentStatusInCluster(segment2, false, 0, 3L, false) + )); + + InputStreamFullResponseHolder responseHolder = new InputStreamFullResponseHolder(new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK)); + byte[] bytesToWrite = mapper.writeValueAsString(segmentStatusInClusterList).getBytes(StandardCharsets.UTF_8); + responseHolder.addChunk(bytesToWrite); + responseHolder.done(); + return (H) responseHolder; + } + }; + + metadataView = new BrokerSegmentMetadataView( + druidLeaderClient, + mapper, + new BrokerSegmentWatcherConfig(), + new BrokerSegmentMetadataCacheConfig() { + @Override + public boolean isMetadataSegmentCacheEnable() + { + return false; + } + }, + cache + ); + druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class); serverInventoryView = EasyMock.createMock(FilteredServerInventoryView.class); schema = new SystemSchema( @@ -283,6 +349,17 @@ public void setUp() throws Exception ); } + @Test + public void test1() throws IOException + { + SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster(segment1, true, 2, 4L, Boolean.TRUE); + byte[] x = mapper.writeValueAsBytes(segmentStatusInCluster); + SegmentStatusInCluster w = mapper.readValue(x, new TypeReference() + { + }); + int p = 1; + } + private final CompactionState expectedCompactionState = new CompactionState( new DynamicPartitionsSpec(null, null), null, @@ -570,17 +647,7 @@ public void testSegmentsTable() throws Exception { final SegmentsTable segmentsTable = new SegmentsTable(metadataView, new ObjectMapper(), authMapper); - final Set segmentTableViews = new HashSet<>(Arrays.asList( - new SegmentTableView(publishedCompactedSegment1, 1L, 0L, 1, 2, 2, true), - new SegmentTableView(publishedCompactedSegment2, 1L, 0L, 0, 2, 0, false), - new SegmentTableView(publishedUncompactedSegment3, 1L, 0L, 1, 2, 2, false), - new SegmentTableView(segment1, 1L, 0L, 1, 2, 2, true), - new SegmentTableView(segment2, 1L, 0L, 1, 2, 0, false) - )); - - EasyMock.expect(metadataView.getSegmentTableView()).andReturn(segmentTableViews.iterator()).once(); - - EasyMock.replay(client, request, responseHolder, responseHandler, metadataView); + EasyMock.replay(client, request, responseHolder, responseHandler); DataContext dataContext = createDataContext(Users.SUPER); final List rows = segmentsTable.scan(dataContext).toList(); rows.sort((Object[] row1, Object[] row2) -> ((Comparable) row1[0]).compareTo(row2[0])); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java index 3f5e9b14f5e9..66457b292f83 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java @@ -433,7 +433,7 @@ public static DruidSchemaCatalog createMockRootSchema( /** * A fake {@link HttpClient} for {@link #createMockSystemSchema}. */ - private static class FakeHttpClient implements HttpClient + public static class FakeHttpClient implements HttpClient { @Override public ListenableFuture go( @@ -458,7 +458,7 @@ public ListenableFuture go( /** * A fake {@link DruidNodeDiscoveryProvider} for {@link #createMockSystemSchema}. */ - private static class FakeDruidNodeDiscoveryProvider extends DruidNodeDiscoveryProvider + public static class FakeDruidNodeDiscoveryProvider extends DruidNodeDiscoveryProvider { private final Map nodeDiscoveries; @@ -484,7 +484,7 @@ public DruidNodeDiscovery getForNodeRole(NodeRole nodeRole) } } - private static class FakeDruidNodeDiscovery implements DruidNodeDiscovery + public static class FakeDruidNodeDiscovery implements DruidNodeDiscovery { private final Set nodes; @@ -493,7 +493,7 @@ private static class FakeDruidNodeDiscovery implements DruidNodeDiscovery this.nodes = new HashSet<>(); } - FakeDruidNodeDiscovery(Map nodes) + public FakeDruidNodeDiscovery(Map nodes) { this.nodes = Sets.newHashSetWithExpectedSize(nodes.size()); nodes.forEach((k, v) -> { diff --git a/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java b/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java index 76abef005422..dfd366250457 100644 --- a/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java +++ b/sql/src/test/java/org/apache/druid/sql/guice/SqlModuleTest.java @@ -31,6 +31,8 @@ import org.apache.druid.client.FilteredServerInventoryView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.client.coordinator.CoordinatorClient; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.client.indexing.IndexingService; import org.apache.druid.client.indexing.NoopOverlordClient; import org.apache.druid.discovery.DruidLeaderClient; @@ -209,6 +211,7 @@ private Injector makeInjectorWithProperties(final Properties props) binder.bind(ResponseContextConfig.class).toInstance(SqlResourceTest.TEST_RESPONSE_CONTEXT_CONFIG); binder.bind(CatalogResolver.class).toInstance(CatalogResolver.NULL_RESOLVER); binder.bind(OverlordClient.class).to(NoopOverlordClient.class); + binder.bind(CoordinatorClient.class).to(NoopCoordinatorClient.class); }, sqlModule, new TestViewManagerModule() From bc8396c0409b9c971cf7f5e4472de6fa431692f4 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 8 Sep 2023 22:12:49 +0530 Subject: [PATCH 11/36] minor change --- .../client/coordinator/CoordinatorClient.java | 2 +- .../metadata/SegmentMetadataCacheConfig.java | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index b6a966c408b0..2f51ca2ec743 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -48,7 +48,7 @@ public interface CoordinatorClient ListenableFuture> fetchUsedSegments(String dataSource, List intervals); /** - * Fetches schema for the given dataSources. + * Fetches information for the given dataSources. */ ListenableFuture> fetchDataSourceInformation(Set datasources); diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index b1932485b1ad..4cf8d0fb0019 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.segment.metadata; import com.fasterxml.jackson.annotation.JsonProperty; From fbab4c88ac7e6f4e83a808fb9076fe7d16897a88 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Fri, 8 Sep 2023 23:07:39 +0530 Subject: [PATCH 12/36] Remove logic to refactor sys segments table building logic --- .../metadata/SegmentMetadataCache.java | 2 +- .../schema/BrokerSegmentMetadataView.java | 355 ------------------ .../druid/sql/calcite/schema/DruidSchema.java | 5 + .../calcite/schema/MetadataSegmentView.java | 244 ++++++++++++ .../sql/calcite/schema/SystemSchema.java | 257 +++++++------ .../org/apache/druid/sql/guice/SqlModule.java | 1 + .../sql/calcite/schema/SystemSchemaTest.java | 111 ++---- .../druid/sql/calcite/util/CalciteTests.java | 47 +-- .../sql/calcite/util/QueryFrameworkUtils.java | 2 +- 9 files changed, 435 insertions(+), 589 deletions(-) delete mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 5483571ce59b..32feb8844179 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -871,7 +871,7 @@ public AvailableSegmentMetadata getAvailableSegmentMetadata(String datasource, S * Returns total number of segments. This method doesn't use the lock intentionally to avoid expensive contention. * As a result, the returned value might be inexact. */ - int getTotalSegments() + public int getTotalSegments() { return totalSegments; } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java deleted file mode 100644 index 38b570b779d1..000000000000 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataView.java +++ /dev/null @@ -1,355 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.sql.calcite.schema; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.collect.ImmutableSortedSet; -import com.google.common.util.concurrent.Uninterruptibles; -import com.google.inject.Inject; -import org.apache.druid.client.BrokerSegmentWatcherConfig; -import org.apache.druid.client.DataSegmentInterner; -import org.apache.druid.client.JsonParserIterator; -import org.apache.druid.client.coordinator.Coordinator; -import org.apache.druid.concurrent.LifecycleLock; -import org.apache.druid.discovery.DruidLeaderClient; -import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.lifecycle.LifecycleStop; -import org.apache.druid.java.util.emitter.EmittingLogger; -import org.apache.druid.metadata.SegmentsMetadataManager; -import org.apache.druid.segment.metadata.AvailableSegmentMetadata; -import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable.SegmentTableView; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.SegmentStatusInCluster; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -/** - * This class polls the Coordinator in background to keep the latest published segments. - * Provides {@link #getSegmentTableView()} for sys segments table view. - * - * This class polls the data from {@link SegmentsMetadataManager} object in the memory of the - * currently leading Coordinator via HTTP queries. - */ -@ManageLifecycle -public class BrokerSegmentMetadataView -{ - private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataView.class); - - private final BrokerSegmentWatcherConfig segmentWatcherConfig; - private final DruidLeaderClient druidLeaderClient; - private final ObjectMapper objectMapper; - private final boolean isMetadataSegmentCacheEnabled; - - /** - * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and - * sys.segments queries return the segments in sorted order based on segmentId. - * - * Volatile since this reference is reassigned in {@code pollSegmentMetadata()} - * and then read in {@code getSegmentMetadata()} - * from other threads. - */ - @MonotonicNonNull - private volatile ImmutableSortedSet segmentMetadata = null; - - private final BrokerSegmentMetadataCache segmentMetadataCache; - - /** - * Caches the replication factor for segment IDs. In case of coordinator restarts or leadership re-elections, - * the coordinator API returns `null` replication factor until load rules are evaluated. - * The cache can be used during these periods to continue serving the previously fetched values. - */ - private final Cache segmentIdToReplicationFactor; - private final ScheduledExecutorService scheduledExec; - private final long pollPeriodInMS; - private final LifecycleLock lifecycleLock = new LifecycleLock(); - private final CountDownLatch segmentMetadataCachePopulated = new CountDownLatch(1); - - @Inject - public BrokerSegmentMetadataView( - final @Coordinator DruidLeaderClient druidLeaderClient, - final ObjectMapper objectMapper, - final BrokerSegmentWatcherConfig segmentWatcherConfig, - final BrokerSegmentMetadataCacheConfig config, - final BrokerSegmentMetadataCache segmentMetadataCache - ) - { - Preconditions.checkNotNull(config, "BrokerSegmentMetadataCacheConfig"); - this.druidLeaderClient = druidLeaderClient; - this.objectMapper = objectMapper; - this.segmentWatcherConfig = segmentWatcherConfig; - - this.isMetadataSegmentCacheEnabled = config.isMetadataSegmentCacheEnable(); - - this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); - this.scheduledExec = Execs.scheduledSingleThreaded("SegmentMetadataView-Cache--%d"); - this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() - .expireAfterAccess(10, TimeUnit.MINUTES) - .build(); - this.segmentMetadataCache = segmentMetadataCache; - } - - @LifecycleStart - public void start() - { - if (!lifecycleLock.canStart()) { - throw new ISE("can't start."); - } - try { - if (isMetadataSegmentCacheEnabled) { - scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); - } - lifecycleLock.started(); - log.info("MetadataSegmentView Started. Configs isMetadataSegmentCacheEnabled [%s]", - isMetadataSegmentCacheEnabled); - } - finally { - lifecycleLock.exitStart(); - } - } - - @LifecycleStop - public void stop() - { - if (!lifecycleLock.canStop()) { - throw new ISE("can't stop."); - } - log.info("MetadataSegmentView is stopping."); - if (isMetadataSegmentCacheEnabled) { - scheduledExec.shutdown(); - } - log.info("MetadataSegmentView Stopped."); - } - - protected Iterator getSegmentTableView() - { - // set of published and realtime segments - final ImmutableSortedSet segments = getSegmentMetadata(); - - // set of available segments and metadata - final Map availableSegmentMetadataMap = segmentMetadataCache.getSegmentMetadataSnapshot(); - final List segmentsTableViews = new ArrayList<>(); - - Set seenSegments = new HashSet<>(); - - for (SegmentStatusInCluster segmentStatusInCluster : segments) { - DataSegment segment = segmentStatusInCluster.getDataSegment(); - SegmentId segmentId = segment.getId(); - AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataMap.get(segmentId); - - long numReplicas = 0L, numRows = 0L, isAvailable = 0L; - if (availableSegmentMetadata != null) { - numReplicas = availableSegmentMetadata.getNumReplicas(); - numRows = availableSegmentMetadata.getNumRows(); - isAvailable = 1L; - } - - // Prefer numRows & realtime status info returned from Coordinator. - if (null != segmentStatusInCluster.getNumRows()) - { - numRows = segmentStatusInCluster.getNumRows(); - } - - long isRealtime = Boolean.TRUE.equals(segmentStatusInCluster.isRealtime()) ? 1 : 0; - - // set of segments returned from coordinator include published and realtime segments - // so realtime segments are not published and vice versa - boolean isPublished = !segmentStatusInCluster.isRealtime(); - - // is_active is true for published segments that are not overshadowed - boolean isActive = isPublished && !segmentStatusInCluster.isOvershadowed(); - - SegmentTableView segmentTableView = new SegmentTableView( - segment, - isAvailable, - isRealtime, - numReplicas, - numRows, - // If the segment is unpublished, we won't have this information yet. - // If the value is null, the load rules might have not evaluated yet, and we don't know the replication factor. - // This should be automatically updated in the next refesh with Coordinator. - segmentStatusInCluster.getReplicationFactor(), - // If the segment is unpublished this value would be false - segmentStatusInCluster.isOvershadowed(), - isPublished, - isActive - ); - - seenSegments.add(segmentId); - segmentsTableViews.add((segmentTableView)); - } - - for (Map.Entry availableSegmentMetadataEntry : availableSegmentMetadataMap.entrySet()) { - if (seenSegments.contains(availableSegmentMetadataEntry.getKey())) { - continue; - } - AvailableSegmentMetadata availableSegmentMetadata = availableSegmentMetadataEntry.getValue(); - - // since all published segments would have been covered in the previous loop - // the curent set of segments must be unpublished - boolean isPublished = false; - - // is_active is true for unpublished segments iff they are realtime - boolean isActive = availableSegmentMetadata.isRealtime() == 1L; - - SegmentTableView segmentTableView = new SegmentTableView( - availableSegmentMetadata.getSegment(), - // segments announced by historicals or realtime tasks are assumed to be available - 1L, - availableSegmentMetadata.isRealtime(), - availableSegmentMetadata.getNumReplicas(), - availableSegmentMetadata.getNumRows(), - // If the segment is unpublished, we won't have this information yet. - null, - // there is an assumption here that unpublished segments are never overshadowed - false, - isPublished, - isActive - ); - - segmentsTableViews.add(segmentTableView); - } - - log.info("Built the segment table view"); - for (SegmentTableView segmentTableView : segmentsTableViews) { - log.info("SegmentTableView is [%s]", segmentTableView); - } - return segmentsTableViews.iterator(); - } - - private void pollSegmentMetadata() - { - log.info("Polling segment metadata from coordinator"); - - segmentMetadata = fetchSegmentMetadata(); - segmentMetadataCachePopulated.countDown(); - } - - private ImmutableSortedSet getSegmentMetadata() - { - if (isMetadataSegmentCacheEnabled) { - Uninterruptibles.awaitUninterruptibly(segmentMetadataCachePopulated); - return segmentMetadata; - } else { - return fetchSegmentMetadata(); - } - } - - private ImmutableSortedSet fetchSegmentMetadata() - { - final Iterator metadataSegments = - querySegmentMetadata(segmentWatcherConfig.getWatchedDataSources()); - - final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); - log.info("Polled segments from coordinator"); - while (metadataSegments.hasNext()) { - final SegmentStatusInCluster segment = metadataSegments.next(); - log.info("This is the polled segmentStatusInCluster %s", segment); - final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); - Integer replicationFactor = segment.getReplicationFactor(); - if (replicationFactor == null) { - replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); - } else { - segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); - } - final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( - interned, - segment.isOvershadowed(), - replicationFactor, - segment.getNumRows(), - segment.isRealtime() - ); - log.info("SegmentStatusInCluster %s", segmentStatusInCluster); - builder.add(segmentStatusInCluster); - } - - return builder.build(); - } - - // Note that coordinator must be up to get segments - private JsonParserIterator querySegmentMetadata( - Set watchedDataSources - ) - { - final StringBuilder queryBuilder = new StringBuilder("/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus"); - - //queryBuilder.append("&includeRealtimeSegments"); - if (watchedDataSources != null && !watchedDataSources.isEmpty()) { - log.debug( - "filtering datasources in published segments based on broker's watchedDataSources[%s]", watchedDataSources); - for (String ds : watchedDataSources) { - queryBuilder.append("&datasources=").append(ds).append("&"); - } - queryBuilder.setLength(queryBuilder.length() - 1); - } - - String query = queryBuilder.toString(); - log.info("query is %s", query); - return SystemSchema.getThingsFromLeaderNode( - query, - new TypeReference() - { - }, - druidLeaderClient, - objectMapper - ); - } - - private class PollTask implements Runnable - { - @Override - public void run() - { - long delayMS = pollPeriodInMS; - try { - final long pollStartTime = System.nanoTime(); - pollSegmentMetadata(); - final long pollEndTime = System.nanoTime(); - final long pollTimeNS = pollEndTime - pollStartTime; - final long pollTimeMS = TimeUnit.NANOSECONDS.toMillis(pollTimeNS); - delayMS = Math.max(pollPeriodInMS - pollTimeMS, 0); - } - catch (Exception e) { - log.makeAlert(e, "Problem polling Coordinator.").emit(); - } - finally { - if (!Thread.currentThread().isInterrupted()) { - scheduledExec.schedule(new PollTask(), delayMS, TimeUnit.MILLISECONDS); - } - } - } - } -} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java index 7f4f626e6ec6..aa1e7965c27e 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java @@ -46,6 +46,11 @@ public DruidSchema( } } + protected BrokerSegmentMetadataCache cache() + { + return segmentMetadataCache; + } + @Override public Table getTable(String name) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java new file mode 100644 index 000000000000..4f8044c48d45 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableSortedSet; +import com.google.common.util.concurrent.Uninterruptibles; +import com.google.inject.Inject; +import org.apache.druid.client.BrokerSegmentWatcherConfig; +import org.apache.druid.client.DataSegmentInterner; +import org.apache.druid.client.JsonParserIterator; +import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.concurrent.LifecycleLock; +import org.apache.druid.discovery.DruidLeaderClient; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.lifecycle.LifecycleStop; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.metadata.SegmentsMetadataManager; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.SegmentStatusInCluster; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * This class polls the Coordinator in background to keep the latest segments. + * Provides {@link #getSegments()} for others to get segments in metadata store. + * + * The difference between this class and {@link SegmentsMetadataManager} is that this class resides + * in Broker's memory, while {@link SegmentsMetadataManager} resides in Coordinator's memory. In + * fact, this class polls the data from {@link SegmentsMetadataManager} object in the memory of the + * currently leading Coordinator via HTTP queries. + */ +@ManageLifecycle +public class MetadataSegmentView +{ + private static final EmittingLogger log = new EmittingLogger(MetadataSegmentView.class); + + private final DruidLeaderClient coordinatorDruidLeaderClient; + private final ObjectMapper jsonMapper; + private final BrokerSegmentWatcherConfig segmentWatcherConfig; + + private final boolean isCacheEnabled; + /** + * Use {@link ImmutableSortedSet} so that the order of segments is deterministic and + * sys.segments queries return the segments in sorted order based on segmentId. + * + * Volatile since this reference is reassigned in {@code poll()} and then read in {@code getPublishedSegments()} + * from other threads. + */ + @MonotonicNonNull + private volatile ImmutableSortedSet publishedSegments = null; + /** + * Caches the replication factor for segment IDs. In case of coordinator restarts or leadership re-elections, the coordinator API returns `null` replication factor until load rules are evaluated. + * The cache can be used during these periods to continue serving the previously fetched values. + */ + private final Cache segmentIdToReplicationFactor; + private final ScheduledExecutorService scheduledExec; + private final long pollPeriodInMS; + private final LifecycleLock lifecycleLock = new LifecycleLock(); + private final CountDownLatch cachePopulated = new CountDownLatch(1); + + @Inject + public MetadataSegmentView( + final @Coordinator DruidLeaderClient druidLeaderClient, + final ObjectMapper jsonMapper, + final BrokerSegmentWatcherConfig segmentWatcherConfig, + final BrokerSegmentMetadataCacheConfig config + ) + { + Preconditions.checkNotNull(config, "BrokerSegmentMetadataCacheConfig"); + this.coordinatorDruidLeaderClient = druidLeaderClient; + this.jsonMapper = jsonMapper; + this.segmentWatcherConfig = segmentWatcherConfig; + this.isCacheEnabled = config.isMetadataSegmentCacheEnable(); + this.pollPeriodInMS = config.getMetadataSegmentPollPeriod(); + this.scheduledExec = Execs.scheduledSingleThreaded("MetadataSegmentView-Cache--%d"); + this.segmentIdToReplicationFactor = CacheBuilder.newBuilder() + .expireAfterAccess(10, TimeUnit.MINUTES) + .build(); + } + + @LifecycleStart + public void start() + { + if (!lifecycleLock.canStart()) { + throw new ISE("can't start."); + } + try { + if (isCacheEnabled) { + scheduledExec.schedule(new PollTask(), pollPeriodInMS, TimeUnit.MILLISECONDS); + } + lifecycleLock.started(); + log.info("MetadataSegmentView is started."); + } + finally { + lifecycleLock.exitStart(); + } + } + + @LifecycleStop + public void stop() + { + if (!lifecycleLock.canStop()) { + throw new ISE("can't stop."); + } + log.info("MetadataSegmentView is stopping."); + if (isCacheEnabled) { + scheduledExec.shutdown(); + } + log.info("MetadataSegmentView is stopped."); + } + + private void poll() + { + log.info("Polling segments from coordinator"); + final JsonParserIterator metadataSegments = getMetadataSegments( + coordinatorDruidLeaderClient, + jsonMapper, + segmentWatcherConfig.getWatchedDataSources() + ); + + final ImmutableSortedSet.Builder builder = ImmutableSortedSet.naturalOrder(); + while (metadataSegments.hasNext()) { + final SegmentStatusInCluster segment = metadataSegments.next(); + final DataSegment interned = DataSegmentInterner.intern(segment.getDataSegment()); + Integer replicationFactor = segment.getReplicationFactor(); + if (replicationFactor == null) { + replicationFactor = segmentIdToReplicationFactor.getIfPresent(segment.getDataSegment().getId()); + } else { + segmentIdToReplicationFactor.put(segment.getDataSegment().getId(), segment.getReplicationFactor()); + } + final SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster( + interned, + segment.isOvershadowed(), + replicationFactor, + segment.getNumRows(), + segment.isRealtime() + ); + builder.add(segmentStatusInCluster); + } + publishedSegments = builder.build(); + cachePopulated.countDown(); + } + + Iterator getSegments() + { + if (isCacheEnabled) { + Uninterruptibles.awaitUninterruptibly(cachePopulated); + return publishedSegments.iterator(); + } else { + return getMetadataSegments( + coordinatorDruidLeaderClient, + jsonMapper, + segmentWatcherConfig.getWatchedDataSources() + ); + } + } + + // Note that coordinator must be up to get segments + private JsonParserIterator getMetadataSegments( + DruidLeaderClient coordinatorClient, + ObjectMapper jsonMapper, + Set watchedDataSources + ) + { + StringBuilder queryBuilder = new StringBuilder("/druid/coordinator/v1/metadata/segments?includeOvershadowedStatus&includeRealtimeSegments"); + if (watchedDataSources != null && !watchedDataSources.isEmpty()) { + log.debug( + "Filtering datasources in segments based on broker's watchedDataSources[%s]", watchedDataSources); + final StringBuilder sb = new StringBuilder(); + for (String ds : watchedDataSources) { + sb.append("datasources=").append(ds).append("&"); + } + sb.setLength(sb.length() - 1); + queryBuilder.append("&"); + queryBuilder.append(sb); + } + + return SystemSchema.getThingsFromLeaderNode( + queryBuilder.toString(), + new TypeReference() + { + }, + coordinatorClient, + jsonMapper + ); + } + + private class PollTask implements Runnable + { + @Override + public void run() + { + long delayMS = pollPeriodInMS; + try { + final long pollStartTime = System.nanoTime(); + poll(); + final long pollEndTime = System.nanoTime(); + final long pollTimeNS = pollEndTime - pollStartTime; + final long pollTimeMS = TimeUnit.NANOSECONDS.toMillis(pollTimeNS); + delayMS = Math.max(pollPeriodInMS - pollTimeMS, 0); + } + catch (Exception e) { + log.makeAlert(e, "Problem polling Coordinator.").emit(); + } + finally { + if (!Thread.currentThread().isInterrupted()) { + scheduledExec.schedule(new PollTask(), delayMS, TimeUnit.MILLISECONDS); + } + } + } + } + +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index afa0986467d5..48e3a4a44268 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -27,6 +27,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.FluentIterable; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.common.net.HostAndPort; import com.google.common.util.concurrent.Futures; import com.google.inject.Inject; @@ -63,10 +66,10 @@ import org.apache.druid.java.util.http.client.Request; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; -import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.server.DruidNode; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; @@ -80,6 +83,7 @@ import org.apache.druid.sql.calcite.table.RowSignatures; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.SegmentStatusInCluster; import org.jboss.netty.handler.codec.http.HttpMethod; import javax.annotation.Nullable; @@ -94,6 +98,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; public class SystemSchema extends AbstractSchema @@ -104,10 +109,10 @@ public class SystemSchema extends AbstractSchema private static final String TASKS_TABLE = "tasks"; private static final String SUPERVISOR_TABLE = "supervisors"; - private static final Function> - SEGMENT_STATUS_IN_CLUSTER_RA_GENERATOR = segmentTableView -> + private static final Function> + SEGMENT_STATUS_IN_CLUSTER_RA_GENERATOR = segment -> Collections.singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply( - segmentTableView.getSegment().getDataSource()) + segment.getDataSegment().getDataSource()) ); private static final Function> SEGMENT_RA_GENERATOR = @@ -115,6 +120,21 @@ public class SystemSchema extends AbstractSchema segment.getDataSource()) ); + private static final long REPLICATION_FACTOR_UNKNOWN = -1L; + + /** + * Booleans constants represented as long type, + * where 1 = true and 0 = false to make it easy to count number of segments + * which are published, available etc. + */ + private static final long IS_ACTIVE_FALSE = 0L; + private static final long IS_ACTIVE_TRUE = 1L; + private static final long IS_PUBLISHED_FALSE = 0L; + private static final long IS_PUBLISHED_TRUE = 1L; + private static final long IS_AVAILABLE_TRUE = 1L; + private static final long IS_OVERSHADOWED_FALSE = 0L; + private static final long IS_OVERSHADOWED_TRUE = 1L; + static final RowSignature SEGMENTS_SIGNATURE = RowSignature .builder() .add("segment_id", ColumnType.STRING) @@ -192,7 +212,8 @@ public class SystemSchema extends AbstractSchema @Inject public SystemSchema( - final BrokerSegmentMetadataView metadataView, + final DruidSchema druidSchema, + final MetadataSegmentView metadataView, final TimelineServerView serverView, final FilteredServerInventoryView serverInventoryView, final AuthorizerMapper authorizerMapper, @@ -204,7 +225,7 @@ public SystemSchema( { Preconditions.checkNotNull(serverView, "serverView"); this.tableMap = ImmutableMap.of( - SEGMENTS_TABLE, new SegmentsTable(metadataView, jsonMapper, authorizerMapper), + SEGMENTS_TABLE, new SegmentsTable(druidSchema, metadataView, jsonMapper, authorizerMapper), SERVERS_TABLE, new ServersTable(druidNodeDiscoveryProvider, serverInventoryView, authorizerMapper, overlordClient, coordinatorDruidLeaderClient), SERVER_SEGMENTS_TABLE, new ServerSegmentsTable(serverView, authorizerMapper), TASKS_TABLE, new TasksTable(overlordClient, authorizerMapper), @@ -223,16 +244,19 @@ public Map getTableMap() */ static class SegmentsTable extends AbstractTable implements ScannableTable { + private final DruidSchema druidSchema; private final ObjectMapper jsonMapper; private final AuthorizerMapper authorizerMapper; - private final BrokerSegmentMetadataView metadataView; + private final MetadataSegmentView metadataView; public SegmentsTable( - BrokerSegmentMetadataView metadataView, + DruidSchema druidSchemna, + MetadataSegmentView metadataView, ObjectMapper jsonMapper, AuthorizerMapper authorizerMapper ) { + this.druidSchema = druidSchemna; this.metadataView = metadataView; this.jsonMapper = jsonMapper; this.authorizerMapper = authorizerMapper; @@ -253,12 +277,50 @@ public TableType getJdbcTableType() @Override public Enumerable scan(DataContext root) { - final Iterator segmentTableView = metadataView.getSegmentTableView(); + //get available segments from druidSchema + final Map availableSegmentMetadata = + druidSchema.cache().getSegmentMetadataSnapshot(); + final Iterator> availableSegmentEntries = + availableSegmentMetadata.entrySet().iterator(); + + // in memory map to store segment data from available segments + final Map partialSegmentDataMap = + Maps.newHashMapWithExpectedSize(druidSchema.cache().getTotalSegments()); + for (AvailableSegmentMetadata h : availableSegmentMetadata.values()) { + PartialSegmentData partialSegmentData = + new PartialSegmentData(IS_AVAILABLE_TRUE, h.isRealtime(), h.getNumReplicas(), h.getNumRows()); + partialSegmentDataMap.put(h.getSegment().getId(), partialSegmentData); + } + + // Get segments from metadata segment cache (if enabled in SQL planner config), else directly from + // Coordinator. + // this may include both published and realtime segments. + final Iterator metadataStoreSegments = metadataView.getSegments(); + + final Set segmentsAlreadySeen = Sets.newHashSetWithExpectedSize(druidSchema.cache().getTotalSegments()); - final FluentIterable allSegments = FluentIterable - .from(() -> getAuthorizedPublishedSegments(segmentTableView, root)) + final FluentIterable publishedSegments = FluentIterable + .from(() -> getAuthorizedPublishedSegments(metadataStoreSegments, root)) .transform(val -> { - final DataSegment segment = val.getSegment(); + final DataSegment segment = val.getDataSegment(); + segmentsAlreadySeen.add(segment.getId()); + final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(segment.getId()); + long numReplicas = 0L, numRows = 0L, isRealtime = 0L, isAvailable = 0L; + if (partialSegmentData != null) { + numReplicas = partialSegmentData.getNumReplicas(); + numRows = partialSegmentData.getNumRows(); + isAvailable = partialSegmentData.isAvailable(); + } + + isRealtime = Boolean.TRUE.equals(val.isRealtime()) ? 1 : 0; + + // set of segments returned from coordinator include published and realtime segments + // so realtime segments are not published and vice versa + boolean isPublished = !val.isRealtime(); + + // is_active is true for published segments that are not overshadowed + boolean isActive = isPublished && !val.isOvershadowed(); + try { return new Object[]{ segment.getId(), @@ -268,20 +330,64 @@ public Enumerable scan(DataContext root) segment.getSize(), segment.getVersion(), (long) segment.getShardSpec().getPartitionNum(), - val.getNumReplicas(), - val.getNumRows(), - val.isActive(), - val.isPublished(), - val.isAvailable(), - val.isRealtime(), - val.isOvershadowed(), + numReplicas, + numRows, + isActive ? IS_ACTIVE_TRUE : IS_ACTIVE_FALSE, + isPublished ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE, + isAvailable, + isRealtime, + val.isOvershadowed() ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE, segment.getShardSpec() == null ? null : jsonMapper.writeValueAsString(segment.getShardSpec()), segment.getDimensions() == null ? null : jsonMapper.writeValueAsString(segment.getDimensions()), segment.getMetrics() == null ? null : jsonMapper.writeValueAsString(segment.getMetrics()), - segment.getLastCompactionState() == null - ? null - : jsonMapper.writeValueAsString(segment.getLastCompactionState()), - val.getReplicationFactor() + segment.getLastCompactionState() == null ? null : jsonMapper.writeValueAsString(segment.getLastCompactionState()), + // If the segment is unpublished, we won't have this information yet. + // If the value is null, the load rules might have not evaluated yet, and we don't know the replication factor. + // This should be automatically updated in the next refesh with Coordinator. + val.getReplicationFactor() == null ? REPLICATION_FACTOR_UNKNOWN : (long) val.getReplicationFactor() + }; + } + catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + + final FluentIterable availableSegments = FluentIterable + .from(() -> getAuthorizedAvailableSegments( + availableSegmentEntries, + root + )) + .transform(val -> { + if (segmentsAlreadySeen.contains(val.getKey())) { + return null; + } + final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(val.getKey()); + final long numReplicas = partialSegmentData == null ? 0L : partialSegmentData.getNumReplicas(); + try { + return new Object[]{ + val.getKey(), + val.getKey().getDataSource(), + val.getKey().getInterval().getStart().toString(), + val.getKey().getInterval().getEnd().toString(), + val.getValue().getSegment().getSize(), + val.getKey().getVersion(), + (long) val.getValue().getSegment().getShardSpec().getPartitionNum(), + numReplicas, + val.getValue().getNumRows(), + // is_active is true for unpublished segments iff they are realtime + val.getValue().isRealtime() /* is_active */, + // is_published is false for unpublished segments + IS_PUBLISHED_FALSE, + // is_available is assumed to be always true for segments announced by historicals or realtime tasks + IS_AVAILABLE_TRUE, + val.getValue().isRealtime(), + IS_OVERSHADOWED_FALSE, + // there is an assumption here that unpublished segments are never overshadowed + val.getValue().getSegment().getShardSpec() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getShardSpec()), + val.getValue().getSegment().getDimensions() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getDimensions()), + val.getValue().getSegment().getMetrics() == null ? null : jsonMapper.writeValueAsString(val.getValue().getSegment().getMetrics()), + null, // unpublished segments from realtime tasks will not be compacted yet + REPLICATION_FACTOR_UNKNOWN // If the segment is unpublished, we won't have this information yet. }; } catch (JsonProcessingException e) { @@ -289,12 +395,16 @@ public Enumerable scan(DataContext root) } }); + final Iterable allSegments = Iterables.unmodifiableIterable( + Iterables.concat(publishedSegments, availableSegments) + ); return Linq4j.asEnumerable(allSegments).where(Objects::nonNull); + } - private Iterator getAuthorizedPublishedSegments( - Iterator it, + private Iterator getAuthorizedPublishedSegments( + Iterator it, DataContext root ) { @@ -303,7 +413,7 @@ private Iterator getAuthorizedPublishedSegments( "authenticationResult in dataContext" ); - final Iterable authorizedSegments = AuthorizationUtils + final Iterable authorizedSegments = AuthorizationUtils .filterAuthorizedResources( authenticationResult, () -> it, @@ -339,67 +449,35 @@ private Iterator> getAuthorizedAvaila return authorizedSegments.iterator(); } - protected static class SegmentTableView + private static class PartialSegmentData { - /** - * Booleans constants represented as long type, - * where 1 = true and 0 = false to make it easy to count number of segments - * which are published, available etc. - */ - private static final long IS_ACTIVE_FALSE = 0L; - private static final long IS_ACTIVE_TRUE = 1L; - private static final long IS_PUBLISHED_FALSE = 0L; - private static final long IS_PUBLISHED_TRUE = 1L; - private static final long IS_OVERSHADOWED_FALSE = 0L; - private static final long IS_OVERSHADOWED_TRUE = 1L; - private static final long REPLICATION_FACTOR_UNKNOWN = -1L; - - private final DataSegment segment; - private final long available; - private final long realtime; + private final long isAvailable; + private final long isRealtime; private final long numReplicas; private final long numRows; - private final long replicationFactor; - private final long overshadowed; - private final long published; - private final long active; - - public SegmentTableView( - DataSegment segment, - long isAvailable, - long realtime, - long numReplicas, - long numRows, - Integer replicationFactor, - boolean overshadowed, - boolean isPublished, - boolean active + + public PartialSegmentData( + final long isAvailable, + final long isRealtime, + final long numReplicas, + final long numRows ) + { - this.segment = segment; - this.available = isAvailable; - this.realtime = realtime; + this.isAvailable = isAvailable; + this.isRealtime = isRealtime; this.numReplicas = numReplicas; this.numRows = numRows; - this.replicationFactor = (null == replicationFactor) ? REPLICATION_FACTOR_UNKNOWN : (long) replicationFactor; - this.overshadowed = overshadowed ? IS_OVERSHADOWED_TRUE : IS_OVERSHADOWED_FALSE; - this.published = isPublished ? IS_PUBLISHED_TRUE : IS_PUBLISHED_FALSE; - this.active = active ? IS_ACTIVE_TRUE : IS_ACTIVE_FALSE; - } - - public DataSegment getSegment() - { - return segment; } public long isAvailable() { - return available; + return isAvailable; } public long isRealtime() { - return realtime; + return isRealtime; } public long getNumReplicas() @@ -411,41 +489,6 @@ public long getNumRows() { return numRows; } - - public long getReplicationFactor() - { - return replicationFactor; - } - - public long isOvershadowed() - { - return overshadowed; - } - - public long isPublished() - { - return published; - } - - public long isActive() - { - return active; - } - - @Override - public String toString() - { - return "SegmentTableView{" + - "segmentId=" + segment.getId() + - ", isAvailable=" + available + - ", isRealtime=" + realtime + - ", numReplicas=" + numReplicas + - ", numRows=" + numRows + - ", replicationFactor=" + replicationFactor + - ", isOvershadowed=" + overshadowed + - ", isPublished=" + published + - '}'; - } } } diff --git a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java index 63ef71585dd9..56d0d2d5d41f 100644 --- a/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java +++ b/sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java @@ -62,6 +62,7 @@ public class SqlModule implements Module public static final String PROPERTY_SQL_VIEW_MANAGER_TYPE = "druid.sql.viewmanager.type"; public static final String PROPERTY_SQL_SCHEMA_MANAGER_TYPE = "druid.sql.schemamanager.type"; public static final String PROPERTY_SQL_APPROX_COUNT_DISTINCT_CHOICE = "druid.sql.approxCountDistinct.function"; + private Properties props; @Inject diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index a159dcc24e5c..ed7f083dda89 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -19,7 +19,6 @@ package org.apache.druid.sql.calcite.schema; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; @@ -37,12 +36,11 @@ import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.druid.client.BrokerSegmentWatcherConfig; -import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableDruidServer; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.common.config.NullHandling; @@ -64,7 +62,6 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.http.client.Request; -import org.apache.druid.java.util.http.client.response.FullResponseHolder; import org.apache.druid.java.util.http.client.response.HttpResponseHandler; import org.apache.druid.java.util.http.client.response.InputStreamFullResponseHolder; import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; @@ -82,13 +79,16 @@ import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.DruidNode; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.coordinator.BytesAccumulatingResponseHandler; +import org.apache.druid.server.coordinator.simulate.TestServerInventoryView; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; @@ -97,26 +97,18 @@ import org.apache.druid.server.security.AuthorizerMapper; import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.server.security.ResourceType; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable; -import org.apache.druid.sql.calcite.schema.SystemSchema.SegmentsTable.SegmentTableView; import org.apache.druid.sql.calcite.table.RowSignatures; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; -import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.timeline.CompactionState; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.SegmentStatusInCluster; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.easymock.EasyMock; -import org.jboss.netty.handler.codec.http.DefaultHttpResponse; -import org.jboss.netty.handler.codec.http.HttpMethod; import org.jboss.netty.handler.codec.http.HttpResponse; -import org.jboss.netty.handler.codec.http.HttpResponseStatus; -import org.jboss.netty.handler.codec.http.HttpVersion; import org.joda.time.DateTime; import org.junit.AfterClass; import org.junit.Assert; @@ -130,7 +122,6 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -162,6 +153,7 @@ public class SystemSchemaTest extends CalciteTestBase ); private SystemSchema schema; + private SpecificSegmentsQuerySegmentWalker walker; private DruidLeaderClient client; private DruidLeaderClient coordinatorClient; private OverlordClient overlordClient; @@ -170,10 +162,11 @@ public class SystemSchemaTest extends CalciteTestBase private StringFullResponseHolder responseHolder; private BytesAccumulatingResponseHandler responseHandler; private Request request; + private DruidSchema druidSchema; private AuthorizerMapper authMapper; private static QueryRunnerFactoryConglomerate conglomerate; private static Closer resourceCloser; - private BrokerSegmentMetadataView metadataView; + private MetadataSegmentView metadataView; private DruidNodeDiscoveryProvider druidNodeDiscoveryProvider; private FilteredServerInventoryView serverInventoryView; @@ -254,7 +247,7 @@ public void setUp() throws Exception .rows(ROWS3) .buildMMappedIndex(); - SpecificSegmentsQuerySegmentWalker walker = new SpecificSegmentsQuerySegmentWalker(conglomerate) + walker = new SpecificSegmentsQuerySegmentWalker(conglomerate) .add(segment1, index1) .add(segment2, index2) .add(segment3, index3); @@ -272,72 +265,14 @@ public void setUp() throws Exception ), new NoopCoordinatorClient() ); - cache.start(); cache.awaitInitialization(); - - DruidNode coordinatorNode = new DruidNode("test-coordinator", "localhost", false, 8081, null, true, false); - DruidLeaderClient druidLeaderClient = new DruidLeaderClient( - new CalciteTests.FakeHttpClient(), - new CalciteTests.FakeDruidNodeDiscoveryProvider( - ImmutableMap.of( - NodeRole.COORDINATOR, - new CalciteTests.FakeDruidNodeDiscovery(ImmutableMap.of(NodeRole.COORDINATOR, coordinatorNode)) - ) - ), - NodeRole.COORDINATOR, - "/simple/leader" - ) - { - @Override - public String findCurrentLeader() - { - return coordinatorNode.getHostAndPortToUse(); - } - - @Override - public Request makeRequest(HttpMethod httpMethod, String urlPath) throws IOException - { - return new Request(httpMethod, new URL(StringUtils.format("http://%s%s", coordinatorNode.getHostAndPortToUse(), urlPath))); - } - - @Override - public > H go(Request request, HttpResponseHandler responseHandler) - throws IOException - { - List segmentStatusInClusterList = new ArrayList<>(Arrays.asList( - new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, 0L, false), - new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, 0L, false), - new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, 0L, false), - new SegmentStatusInCluster(segment1, true, 2, 3L, false), - new SegmentStatusInCluster(segment2, false, 0, 3L, false) - )); - - InputStreamFullResponseHolder responseHolder = new InputStreamFullResponseHolder(new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK)); - byte[] bytesToWrite = mapper.writeValueAsString(segmentStatusInClusterList).getBytes(StandardCharsets.UTF_8); - responseHolder.addChunk(bytesToWrite); - responseHolder.done(); - return (H) responseHolder; - } - }; - - metadataView = new BrokerSegmentMetadataView( - druidLeaderClient, - mapper, - new BrokerSegmentWatcherConfig(), - new BrokerSegmentMetadataCacheConfig() { - @Override - public boolean isMetadataSegmentCacheEnable() - { - return false; - } - }, - cache - ); - + druidSchema = new DruidSchema(cache, null); + metadataView = EasyMock.createMock(MetadataSegmentView.class); druidNodeDiscoveryProvider = EasyMock.createMock(DruidNodeDiscoveryProvider.class); serverInventoryView = EasyMock.createMock(FilteredServerInventoryView.class); schema = new SystemSchema( + druidSchema, metadataView, serverView, serverInventoryView, @@ -349,17 +284,6 @@ public boolean isMetadataSegmentCacheEnable() ); } - @Test - public void test1() throws IOException - { - SegmentStatusInCluster segmentStatusInCluster = new SegmentStatusInCluster(segment1, true, 2, 4L, Boolean.TRUE); - byte[] x = mapper.writeValueAsBytes(segmentStatusInCluster); - SegmentStatusInCluster w = mapper.readValue(x, new TypeReference() - { - }); - int p = 1; - } - private final CompactionState expectedCompactionState = new CompactionState( new DynamicPartitionsSpec(null, null), null, @@ -645,9 +569,18 @@ public void testGetTableMap() @Test public void testSegmentsTable() throws Exception { - final SegmentsTable segmentsTable = new SegmentsTable(metadataView, new ObjectMapper(), authMapper); + final SegmentsTable segmentsTable = new SegmentsTable(druidSchema, metadataView, new ObjectMapper(), authMapper); + final Set publishedSegments = new HashSet<>(Arrays.asList( + new SegmentStatusInCluster(publishedCompactedSegment1, true, 2, null, false), + new SegmentStatusInCluster(publishedCompactedSegment2, false, 0, null, false), + new SegmentStatusInCluster(publishedUncompactedSegment3, false, 2, null, false), + new SegmentStatusInCluster(segment1, true, 2, null, false), + new SegmentStatusInCluster(segment2, false, 0, null, false) + )); + + EasyMock.expect(metadataView.getSegments()).andReturn(publishedSegments.iterator()).once(); - EasyMock.replay(client, request, responseHolder, responseHandler); + EasyMock.replay(client, request, responseHolder, responseHandler, metadataView); DataContext dataContext = createDataContext(Users.SUPER); final List rows = segmentsTable.scan(dataContext).toList(); rows.sort((Object[] row1, Object[] row2) -> ((Comparable) row1[0]).compareTo(row2[0])); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java index 66457b292f83..fc6ad94fe1aa 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/CalciteTests.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -31,11 +30,8 @@ import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; -import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ServerInventoryView; import org.apache.druid.client.ServerView; -import org.apache.druid.client.TimelineServerView; -import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.client.indexing.NoopOverlordClient; import org.apache.druid.discovery.DiscoveryDruidNode; import org.apache.druid.discovery.DruidLeaderClient; @@ -53,16 +49,12 @@ import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.join.JoinableFactoryWrapper; -import org.apache.druid.segment.join.MapJoinableFactory; -import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.metadata.TestTimelineServerView; -import org.apache.druid.server.SegmentManager; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.DruidNode; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryScheduler; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.AuthConfig; @@ -80,15 +72,12 @@ import org.apache.druid.sql.calcite.planner.PlannerFactory; import org.apache.druid.sql.calcite.run.NativeSqlEngine; import org.apache.druid.sql.calcite.run.SqlEngine; -import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache; import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCacheConfig; -import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataView; import org.apache.druid.sql.calcite.schema.DruidSchema; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; -import org.apache.druid.sql.calcite.schema.PhysicalDatasourceMetadataBuilder; +import org.apache.druid.sql.calcite.schema.MetadataSegmentView; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.timeline.DataSegment; -import org.easymock.EasyMock; import org.joda.time.Duration; import javax.annotation.Nullable; @@ -343,7 +332,7 @@ public static DruidOperatorTable createOperatorTable() } public static SystemSchema createMockSystemSchema( - final QueryLifecycleFactory queryLifecycleFactory, + final DruidSchema druidSchema, final SpecificSegmentsQuerySegmentWalker walker, final AuthorizerMapper authorizerMapper ) @@ -383,29 +372,15 @@ public ListenableFuture findCurrentLeader() } }; - TimelineServerView timelineServerView = new TestTimelineServerView(walker.getSegments()); - return new SystemSchema( - new BrokerSegmentMetadataView( + druidSchema, + new MetadataSegmentView( druidLeaderClient, getJsonMapper(), new BrokerSegmentWatcherConfig(), - BrokerSegmentMetadataCacheConfig.create(), - new BrokerSegmentMetadataCache( - queryLifecycleFactory, - timelineServerView, - BrokerSegmentMetadataCacheConfig.create(), - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder( - new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), - new SegmentManager(EasyMock.createMock(SegmentLoader.class)) - ), - new NoopCoordinatorClient() - ) + BrokerSegmentMetadataCacheConfig.create() ), - timelineServerView, + new TestTimelineServerView(walker.getSegments()), new FakeServerInventoryView(), authorizerMapper, druidLeaderClient, @@ -433,7 +408,7 @@ public static DruidSchemaCatalog createMockRootSchema( /** * A fake {@link HttpClient} for {@link #createMockSystemSchema}. */ - public static class FakeHttpClient implements HttpClient + private static class FakeHttpClient implements HttpClient { @Override public ListenableFuture go( @@ -458,7 +433,7 @@ public ListenableFuture go( /** * A fake {@link DruidNodeDiscoveryProvider} for {@link #createMockSystemSchema}. */ - public static class FakeDruidNodeDiscoveryProvider extends DruidNodeDiscoveryProvider + private static class FakeDruidNodeDiscoveryProvider extends DruidNodeDiscoveryProvider { private final Map nodeDiscoveries; @@ -484,7 +459,7 @@ public DruidNodeDiscovery getForNodeRole(NodeRole nodeRole) } } - public static class FakeDruidNodeDiscovery implements DruidNodeDiscovery + private static class FakeDruidNodeDiscovery implements DruidNodeDiscovery { private final Set nodes; @@ -493,7 +468,7 @@ public static class FakeDruidNodeDiscovery implements DruidNodeDiscovery this.nodes = new HashSet<>(); } - public FakeDruidNodeDiscovery(Map nodes) + FakeDruidNodeDiscovery(Map nodes) { this.nodes = Sets.newHashSetWithExpectedSize(nodes.size()); nodes.forEach((k, v) -> { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index 9f540d634d06..dc716b2a45c1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -150,7 +150,7 @@ public static DruidSchemaCatalog createMockRootSchema( druidSchemaManager ); SystemSchema systemSchema = - CalciteTests.createMockSystemSchema(createMockQueryLifecycleFactory(walker, conglomerate), walker, authorizerMapper); + CalciteTests.createMockSystemSchema(druidSchema, walker, authorizerMapper); LookupSchema lookupSchema = createMockLookupSchema(injector); ViewSchema viewSchema = viewManager != null ? new ViewSchema(viewManager) : null; From 151b0b1f00707d8562da956e2715c986613f6709 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 9 Sep 2023 18:25:58 +0530 Subject: [PATCH 13/36] undo changes in SegmentsMetadataManager --- .../org/apache/druid/metadata/SegmentsMetadataManager.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java index eecf8e036195..a774afcd47b3 100644 --- a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java +++ b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java @@ -33,6 +33,11 @@ import java.util.List; import java.util.Set; +/** + * The difference between this class and org.apache.druid.sql.calcite.schema.MetadataSegmentView is that this class + * resides in Coordinator's memory, while org.apache.druid.sql.calcite.schema.MetadataSegmentView resides in Broker's + * memory. + */ public interface SegmentsMetadataManager { void startPollingDatabasePeriodically(); From 8dbea5b9472c47fce42ee54c325ac648a263b00a Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Sat, 9 Sep 2023 21:51:17 +0530 Subject: [PATCH 14/36] Minor code changes and add multiple tests --- .../metadata/SegmentMetadataCache.java | 18 +-- .../druid/server/http/MetadataResource.java | 2 +- .../CoordinatorClientImplTest.java | 30 +++++ .../metadata/SegmentMetadataCacheCommon.java | 13 +- .../server/http/MetadataResourceTest.java | 58 +++++++- .../schema/BrokerSegmentMetadataCache.java | 20 ++- .../BrokerSegmentMetadataCacheConfigTest.java | 94 +++++++++++++ ...PhysicalDataSourceMetadataBuilderTest.java | 126 ++++++++++++++++++ 8 files changed, 321 insertions(+), 40 deletions(-) create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 32feb8844179..704fc52de5f8 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -438,7 +438,7 @@ public void rebuildDatasource(String dataSource) if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { - log.info("[%s] signature is unchanged.", dataSource); + log.debug("[%s] signature is unchanged.", dataSource); } } @@ -459,7 +459,7 @@ public DataSourceInformation getDatasource(String name) return tables.get(name); } - public Map getDataSourceSchemaMap() + public Map getDataSourceInformationMap() { return ImmutableMap.copyOf(tables); } @@ -877,7 +877,7 @@ public int getTotalSegments() } @VisibleForTesting - public TreeSet getSegmentsNeedingRefresh() + TreeSet getSegmentsNeedingRefresh() { synchronized (lock) { return segmentsNeedingRefresh; @@ -885,7 +885,7 @@ public TreeSet getSegmentsNeedingRefresh() } @VisibleForTesting - public TreeSet getMutableSegments() + TreeSet getMutableSegments() { synchronized (lock) { return mutableSegments; @@ -893,19 +893,13 @@ public TreeSet getMutableSegments() } @VisibleForTesting - public Set getDataSourcesNeedingRebuild() + Set getDataSourcesNeedingRebuild() { synchronized (lock) { return dataSourcesNeedingRebuild; } } - Object getLock() - { - return lock; - } - - /** * Execute a SegmentMetadata query and return a {@link Sequence} of {@link SegmentAnalysis}. * @@ -951,7 +945,7 @@ protected Sequence runSegmentMetadataQuery( } @VisibleForTesting - static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) + public static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) { final RowSignature.Builder rowSignatureBuilder = RowSignature.builder(); for (Map.Entry entry : analysis.getColumns().entrySet()) { diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 1e86954ea2fe..447fac8784aa 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -369,7 +369,7 @@ public Response getDataSourceInformation( if (null == segmentMetadataCache) { return Response.status(Response.Status.NOT_FOUND).build(); } - Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceSchemaMap(); + Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceInformationMap(); List results = new ArrayList<>(); List dataSourcesToRetain = (null == dataSources) ? new ArrayList<>(dataSourceSchemaMap.keySet()) : dataSources; diff --git a/server/src/test/java/org/apache/druid/client/coordinator/CoordinatorClientImplTest.java b/server/src/test/java/org/apache/druid/client/coordinator/CoordinatorClientImplTest.java index f48e21327a0b..e418c2772318 100644 --- a/server/src/test/java/org/apache/druid/client/coordinator/CoordinatorClientImplTest.java +++ b/server/src/test/java/org/apache/druid/client/coordinator/CoordinatorClientImplTest.java @@ -28,6 +28,9 @@ import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.rpc.MockServiceClient; import org.apache.druid.rpc.RequestBuilder; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.jboss.netty.handler.codec.http.HttpMethod; @@ -168,4 +171,31 @@ public void test_fetchUsedSegments() throws Exception coordinatorClient.fetchUsedSegments("xyz", intervals).get() ); } + + @Test + public void test_fetchDataSourceInformation() throws Exception + { + String foo = "foo"; + + DataSourceInformation fooInfo = new DataSourceInformation( + "foo", + RowSignature.builder() + .add("d1", ColumnType.FLOAT) + .add("d2", ColumnType.DOUBLE) + .build() + ); + + serviceClient.expectAndRespond( + new RequestBuilder(HttpMethod.POST, "/druid/coordinator/v1/metadata/dataSourceInformation") + .jsonContent(jsonMapper, Collections.singletonList(foo)), + HttpResponseStatus.OK, + ImmutableMap.of(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON), + jsonMapper.writeValueAsBytes(Collections.singletonList(fooInfo)) + ); + + Assert.assertEquals( + Collections.singletonList(fooInfo), + coordinatorClient.fetchDataSourceInformation(Collections.singleton(foo)).get() + ); + } } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index a391ba9fb74c..0bd83f88f2d2 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -86,8 +86,10 @@ public abstract class SegmentMetadataCacheCommon static QueryRunnerFactoryConglomerate conglomerate; static Closer resourceCloser; - static QueryToolChestWarehouse queryToolChestWarehouse; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @BeforeClass public static void setUpClass() @@ -110,15 +112,6 @@ public static void tearDownClass() throws IOException resourceCloser.close(); } - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); - - - @Before - public void setUpCommon() - { - } - InputRow createRow(final ImmutableMap map) { return MapInputRowParser.parse(FOO_SCHEMA, (Map) map); diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index f24cc7bd2c76..141c0ddc5c39 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -24,13 +24,15 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import org.apache.druid.client.DataSourcesSnapshot; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.metadata.SegmentsMetadataManager; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.coordinator.CreateDataSegments; import org.apache.druid.server.coordinator.DruidCoordinator; @@ -47,10 +49,10 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.core.Response; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; public class MetadataResourceTest { @@ -126,7 +128,7 @@ public void testGetAllSegmentsWithOvershadowedStatus() { Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus", null); - final List resultList = extractSegmentStatusList(response); + final List resultList = extractResponseList(response); Assert.assertEquals(resultList.size(), 4); Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, null, false), resultList.get(0)); Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, null, false), resultList.get(1)); @@ -223,7 +225,7 @@ public void testGetAllSegmentsIncludingRealtime() Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus", "includeRealtimeSegments"); - final List resultList = extractSegmentStatusList(response); + final List resultList = extractResponseList(response); Assert.assertEquals(resultList.size(), 6); Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 20L, false), resultList.get(0)); Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 30L, false), resultList.get(1)); @@ -234,6 +236,50 @@ public void testGetAllSegmentsIncludingRealtime() Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[1], false, null, 40L, true), resultList.get(5)); } + @Test + public void testGetDataSourceInformation() { + SegmentMetadataCache segmentMetadataCache = Mockito.mock(SegmentMetadataCache.class); + Map dataSourceInformationMap = new HashMap<>(); + + dataSourceInformationMap.put( + DATASOURCE1, + new DataSourceInformation( + DATASOURCE1, + RowSignature.builder() + .add("c1", ColumnType.FLOAT) + .add("c2", ColumnType.DOUBLE) + .build() + ) + ); + + dataSourceInformationMap.put( + "datasource2", + new DataSourceInformation( + "datasource2", + RowSignature.builder() + .add("d1", ColumnType.FLOAT) + .add("d2", ColumnType.DOUBLE) + .build() + ) + ); + + Mockito.doReturn(dataSourceInformationMap).when(segmentMetadataCache).getDataSourceInformationMap(); + + metadataResource = new MetadataResource( + segmentsMetadataManager, + storageCoordinator, + AuthTestUtils.TEST_AUTHORIZER_MAPPER, + coordinator, + segmentMetadataCache + ); + + Response response = metadataResource.getDataSourceInformation(Collections.singletonList(DATASOURCE1)); + + List dataSourceInformations = extractResponseList(response); + Assert.assertEquals(dataSourceInformations.size(), 1); + Assert.assertEquals(dataSourceInformations.get(0), dataSourceInformationMap.get(DATASOURCE1)); + } + @Test public void testGetSegment() { @@ -259,10 +305,10 @@ public void testGetSegment() ); } - private List extractSegmentStatusList(Response response) + private List extractResponseList(Response response) { return Lists.newArrayList( - (Iterable) response.getEntity() + (Iterable) response.getEntity() ); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 0a0428ad1d50..8a037e7cd039 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -87,43 +87,41 @@ public void refresh(final Set segmentsToRefresh, final Set da segmentsToRefresh.forEach(segment -> dataSourcesToQuery.add(segment.getDataSource())); - Map polledDataSourceSchema = new HashMap<>(); + Map polledDataSourceMetadata = new HashMap<>(); - // Fetch dataSource schema from the Coordinator + // Fetch dataSource information from the Coordinator try { FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) - .forEach(item -> polledDataSourceSchema.put( + .forEach(item -> polledDataSourceMetadata.put( item.getDatasource(), physicalDatasourceMetadataBuilder.build(item) )); } catch (Exception e) { - log.error("Exception querying coordinator for schema"); + log.warn(e, "Exception querying coordinator to fetch dataSourceInformation."); } - log.info("Queried ds schema are [%s]", polledDataSourceSchema); - - tables.putAll(polledDataSourceSchema); + tables.putAll(polledDataSourceMetadata); // Remove segments of the dataSource from refresh list for which we received schema from the Coordinator. segmentsToRefresh.forEach(segment -> { - if (polledDataSourceSchema.containsKey(segment.getDataSource())) { + if (polledDataSourceMetadata.containsKey(segment.getDataSource())) { segmentsToRefresh.remove(segment); } }); - // Refresh the segments. + // Refresh the remaining segments. final Set refreshed = refreshSegments(segmentsToRefresh); synchronized (lock) { // Add missing segments back to the refresh list. - getSegmentsNeedingRefresh().addAll(Sets.difference(segmentsToRefresh, refreshed)); + segmentsNeedingRefresh.addAll(Sets.difference(segmentsToRefresh, refreshed)); // Compute the list of dataSources to rebuild tables for. dataSourcesToRebuild.addAll(dataSourcesNeedingRebuild); refreshed.forEach(segment -> dataSourcesToRebuild.add(segment.getDataSource())); // Remove those dataSource for which we received schema from the Coordinator. - dataSourcesToRebuild.removeAll(polledDataSourceSchema.keySet()); + dataSourcesToRebuild.removeAll(polledDataSourceMetadata.keySet()); dataSourcesNeedingRebuild.clear(); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java new file mode 100644 index 000000000000..25e6096bb190 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Injector; +import org.apache.druid.guice.GuiceInjectors; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.guice.JsonConfigurator; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.sql.calcite.planner.CalcitePlannerModule; +import org.joda.time.Period; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Properties; + +public class BrokerSegmentMetadataCacheConfigTest +{ + + private static final String CONFIG_BASE = CalcitePlannerModule.CONFIG_BASE; + + @Test + public void testDefaultConfig() + { + final Injector injector = createInjector(); + final JsonConfigProvider provider = JsonConfigProvider.of( + CONFIG_BASE, + BrokerSegmentMetadataCacheConfig.class + ); + + final Properties properties = new Properties(); + provider.inject(properties, injector.getInstance(JsonConfigurator.class)); + final BrokerSegmentMetadataCacheConfig config = provider.get(); + Assert.assertTrue(config.isAwaitInitializationOnStart()); + Assert.assertTrue(config.isMetadataSegmentCacheEnable()); + Assert.assertEquals(Period.minutes(1), config.getMetadataRefreshPeriod()); + Assert.assertEquals(new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); + } + + @Test + public void testCustomizedConfig() + { + final Injector injector = createInjector(); + final JsonConfigProvider provider = JsonConfigProvider.of( + CONFIG_BASE, + BrokerSegmentMetadataCacheConfig.class + ); + final Properties properties = new Properties(); + properties.setProperty( + CONFIG_BASE + ".metadataColumnTypeMergePolicy", + "latestInterval" + ); + properties.setProperty(CONFIG_BASE + ".metadataRefreshPeriod", "PT2M"); + properties.setProperty(CONFIG_BASE + ".awaitInitializationOnStart", "false"); + provider.inject(properties, injector.getInstance(JsonConfigurator.class)); + final BrokerSegmentMetadataCacheConfig config = provider.get(); + Assert.assertFalse(config.isAwaitInitializationOnStart()); + Assert.assertTrue(config.isMetadataSegmentCacheEnable()); + Assert.assertEquals(Period.minutes(2), config.getMetadataRefreshPeriod()); + Assert.assertEquals( + new SegmentMetadataCache.FirstTypeMergePolicy(), + config.getMetadataColumnTypeMergePolicy() + ); + } + + private Injector createInjector() + { + return GuiceInjectors.makeStartupInjectorWithModules( + ImmutableList.of( + binder -> { + JsonConfigProvider.bind(binder, CONFIG_BASE, BrokerSegmentMetadataCacheConfig.class); + } + ) + ); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java new file mode 100644 index 000000000000..dc670af203e4 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.google.common.collect.Sets; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.GlobalTableDataSource; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.join.JoinConditionAnalysis; +import org.apache.druid.segment.join.Joinable; +import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.DataSourceInformation; +import org.apache.druid.server.SegmentManager; +import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; + +public class PhysicalDataSourceMetadataBuilderTest +{ + private CountDownLatch getDatasourcesLatch = new CountDownLatch(1); + + private Set segmentDataSourceNames; + private Set joinableDataSourceNames; + private SegmentManager segmentManager; + private JoinableFactory globalTableJoinable; + + private PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; + + @Before + public void setUp() throws Exception + { + segmentDataSourceNames = Sets.newConcurrentHashSet(); + joinableDataSourceNames = Sets.newConcurrentHashSet(); + segmentManager = new SegmentManager(EasyMock.createMock(SegmentLoader.class)) + { + @Override + public Set getDataSourceNames() + { + getDatasourcesLatch.countDown(); + return segmentDataSourceNames; + } + }; + + globalTableJoinable = new JoinableFactory() + { + @Override + public boolean isDirectlyJoinable(DataSource dataSource) + { + return dataSource instanceof GlobalTableDataSource && + joinableDataSourceNames.contains(((GlobalTableDataSource) dataSource).getName()); + } + + @Override + public Optional build( + DataSource dataSource, + JoinConditionAnalysis condition + ) + { + return Optional.empty(); + } + }; + + physicalDatasourceMetadataBuilder = new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager); + } + + @Test + public void testBuild() { + segmentDataSourceNames.add("foo"); + joinableDataSourceNames.add("foo"); + + DataSourceInformation foo = + new DataSourceInformation( + "foo", + RowSignature.builder() + .add("c1", ColumnType.FLOAT) + .add("c2", ColumnType.DOUBLE) + .build() + ); + + DataSourceInformation bar = + new DataSourceInformation( + "bar", + RowSignature.builder() + .add("d1", ColumnType.FLOAT) + .add("d2", ColumnType.DOUBLE) + .build() + ); + + DatasourceTable.PhysicalDatasourceMetadata fooDs = physicalDatasourceMetadataBuilder.build(foo); + Assert.assertTrue(fooDs.isJoinable()); + Assert.assertTrue(fooDs.isBroadcast()); + Assert.assertEquals(fooDs.dataSource().getName(), foo.getDatasource()); + Assert.assertEquals(fooDs.rowSignature(), foo.getRowSignature()); + + DatasourceTable.PhysicalDatasourceMetadata barDs = physicalDatasourceMetadataBuilder.build(bar); + Assert.assertFalse(barDs.isJoinable()); + Assert.assertFalse(barDs.isBroadcast()); + Assert.assertEquals(barDs.dataSource().getName(), bar.getDatasource()); + Assert.assertEquals(barDs.rowSignature(), bar.getRowSignature()); + } +} From e630b9b730598ed34623e211489345830abfd313 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 15:16:51 +0530 Subject: [PATCH 15/36] Add test for QueryableCoordinatorServerViewTest --- .../QueryableCoordinatorServerViewTest.java | 398 ++++++++++++++++++ 1 file changed, 398 insertions(+) create mode 100644 server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java diff --git a/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java new file mode 100644 index 000000000000..0bb391165feb --- /dev/null +++ b/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.smile.SmileFactory; +import com.fasterxml.jackson.dataformat.smile.SmileGenerator; +import com.google.common.base.Function; +import com.google.common.base.Predicates; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.apache.curator.utils.ZKPaths; +import org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy; +import org.apache.druid.client.selector.RandomServerSelectorStrategy; +import org.apache.druid.curator.CuratorTestBase; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.QueryWatcher; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.segment.TestHelper; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; +import org.apache.druid.server.initialization.ZkPathsConfig; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.TimelineLookup; +import org.apache.druid.timeline.TimelineObjectHolder; +import org.apache.druid.timeline.partition.NoneShardSpec; +import org.apache.druid.timeline.partition.PartitionHolder; +import org.easymock.EasyMock; +import org.joda.time.Interval; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; + +public class QueryableCoordinatorServerViewTest extends CuratorTestBase +{ + private final ObjectMapper jsonMapper; + private final ZkPathsConfig zkPathsConfig; + private final String inventoryPath; + + private CountDownLatch segmentViewInitLatch; + private CountDownLatch segmentAddedLatch; + private CountDownLatch segmentRemovedLatch; + + private BatchServerInventoryView baseView; + private QueryableCoordinatorServerView overlordServerView; + + public QueryableCoordinatorServerViewTest() + { + jsonMapper = TestHelper.makeJsonMapper(); + zkPathsConfig = new ZkPathsConfig(); + inventoryPath = zkPathsConfig.getLiveSegmentsPath(); + } + + @Before + public void setUp() throws Exception + { + setupServerAndCurator(); + curator.start(); + curator.blockUntilConnected(); + } + + @Test + public void testSingleServerAddedRemovedSegment() throws Exception + { + segmentViewInitLatch = new CountDownLatch(1); + segmentAddedLatch = new CountDownLatch(1); + segmentRemovedLatch = new CountDownLatch(1); + + setupViews(); + + final DruidServer druidServer = new DruidServer( + "localhost:1234", + "localhost:1234", + null, + 10000000L, + ServerType.HISTORICAL, + "default_tier", + 0 + ); + + setupZNodeForServer(druidServer, zkPathsConfig, jsonMapper); + + final DataSegment segment = dataSegmentWithIntervalAndVersion("2014-10-20T00:00:00Z/P1D", "v1"); + final int partition = segment.getShardSpec().getPartitionNum(); + final Interval intervals = Intervals.of("2014-10-20T00:00:00Z/P1D"); + announceSegmentForServer(druidServer, segment, zkPathsConfig, jsonMapper); + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); + + TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + List serverLookupRes = (List) timeline.lookup( + intervals + ); + Assert.assertEquals(1, serverLookupRes.size()); + + TimelineObjectHolder actualTimelineObjectHolder = serverLookupRes.get(0); + Assert.assertEquals(intervals, actualTimelineObjectHolder.getInterval()); + Assert.assertEquals("v1", actualTimelineObjectHolder.getVersion()); + + PartitionHolder actualPartitionHolder = actualTimelineObjectHolder.getObject(); + Assert.assertTrue(actualPartitionHolder.isComplete()); + Assert.assertEquals(1, Iterables.size(actualPartitionHolder)); + + SegmentLoadInfo segmentLoadInfo = actualPartitionHolder.iterator().next().getObject(); + Assert.assertFalse(segmentLoadInfo.isEmpty()); + Assert.assertEquals( + druidServer.getMetadata(), + Iterables.getOnlyElement(segmentLoadInfo.toImmutableSegmentLoadInfo().getServers()) + ); + Assert.assertNotNull(timeline.findChunk(intervals, "v1", partition)); + + unannounceSegmentForServer(druidServer, segment); + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + + Assert.assertEquals( + 0, + ((List) timeline.lookup(Intervals.of("2014-10-20T00:00:00Z/P1D"))).size() + ); + Assert.assertNull(timeline.findChunk(intervals, "v1", partition)); + } + + @Test + public void testMultipleServerAddedRemovedSegment() throws Exception + { + segmentViewInitLatch = new CountDownLatch(1); + segmentAddedLatch = new CountDownLatch(5); + + // temporarily set latch count to 1 + segmentRemovedLatch = new CountDownLatch(1); + + setupViews(); + + final List druidServers = Lists.transform( + ImmutableList.of("localhost:0", "localhost:1", "localhost:2", "localhost:3", "localhost:4"), + new Function() + { + @Override + public DruidServer apply(String input) + { + return new DruidServer( + input, + input, + null, + 10000000L, + ServerType.HISTORICAL, + "default_tier", + 0 + ); + } + } + ); + + for (DruidServer druidServer : druidServers) { + setupZNodeForServer(druidServer, zkPathsConfig, jsonMapper); + } + + final List segments = Lists.transform( + ImmutableList.of( + Pair.of("2011-04-01/2011-04-03", "v1"), + Pair.of("2011-04-03/2011-04-06", "v1"), + Pair.of("2011-04-01/2011-04-09", "v2"), + Pair.of("2011-04-06/2011-04-09", "v3"), + Pair.of("2011-04-01/2011-04-02", "v3") + ), new Function, DataSegment>() + { + @Override + public DataSegment apply(Pair input) + { + return dataSegmentWithIntervalAndVersion(input.lhs, input.rhs); + } + } + ); + + for (int i = 0; i < 5; ++i) { + announceSegmentForServer(druidServers.get(i), segments.get(i), zkPathsConfig, jsonMapper); + } + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); + + TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + assertValues( + Arrays.asList( + createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), + createExpected("2011-04-02/2011-04-06", "v2", druidServers.get(2), segments.get(2)), + createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3)) + ), + (List) timeline.lookup( + Intervals.of( + "2011-04-01/2011-04-09" + ) + ) + ); + + // unannounce the segment created by dataSegmentWithIntervalAndVersion("2011-04-01/2011-04-09", "v2") + unannounceSegmentForServer(druidServers.get(2), segments.get(2)); + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + + // renew segmentRemovedLatch since we still have 4 segments to unannounce + segmentRemovedLatch = new CountDownLatch(4); + + timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + assertValues( + Arrays.asList( + createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), + createExpected("2011-04-02/2011-04-03", "v1", druidServers.get(0), segments.get(0)), + createExpected("2011-04-03/2011-04-06", "v1", druidServers.get(1), segments.get(1)), + createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3)) + ), + (List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09")) + ); + + // unannounce all the segments + for (int i = 0; i < 5; ++i) { + // skip the one that was previously unannounced + if (i != 2) { + unannounceSegmentForServer(druidServers.get(i), segments.get(i)); + } + } + Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + + Assert.assertEquals( + 0, + ((List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09"))).size() + ); + } + + private void unannounceSegmentForServer(DruidServer druidServer, DataSegment segment) throws Exception + { + curator + .delete() + .guaranteed() + .forPath(ZKPaths.makePath(inventoryPath, druidServer.getHost(), segment.getId().toString())); + } + + private Pair>> createExpected( + String intervalStr, + String version, + DruidServer druidServer, + DataSegment segment + ) + { + return Pair.of(Intervals.of(intervalStr), Pair.of(version, Pair.of(druidServer, segment))); + } + + private void assertValues( + List>>> expected, List actual + ) + { + Assert.assertEquals(expected.size(), actual.size()); + + for (int i = 0; i < expected.size(); ++i) { + Pair>> expectedPair = expected.get(i); + TimelineObjectHolder actualTimelineObjectHolder = actual.get(i); + + Assert.assertEquals(expectedPair.lhs, actualTimelineObjectHolder.getInterval()); + Assert.assertEquals(expectedPair.rhs.lhs, actualTimelineObjectHolder.getVersion()); + + PartitionHolder actualPartitionHolder = actualTimelineObjectHolder.getObject(); + Assert.assertTrue(actualPartitionHolder.isComplete()); + Assert.assertEquals(1, Iterables.size(actualPartitionHolder)); + + SegmentLoadInfo segmentLoadInfo = actualPartitionHolder.iterator().next().getObject(); + Assert.assertFalse(segmentLoadInfo.isEmpty()); + Assert.assertEquals(expectedPair.rhs.rhs.lhs.getMetadata(), + Iterables.getOnlyElement(segmentLoadInfo.toImmutableSegmentLoadInfo().getServers())); + } + } + + private void setupViews() throws Exception + { + baseView = new BatchServerInventoryView( + zkPathsConfig, + curator, + jsonMapper, + Predicates.alwaysTrue(), + "test" + ) + { + @Override + public void registerSegmentCallback(Executor exec, final SegmentCallback callback) + { + super.registerSegmentCallback( + exec, + new SegmentCallback() + { + @Override + public CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) + { + CallbackAction res = callback.segmentAdded(server, segment); + segmentAddedLatch.countDown(); + return res; + } + + @Override + public CallbackAction segmentRemoved(DruidServerMetadata server, DataSegment segment) + { + CallbackAction res = callback.segmentRemoved(server, segment); + segmentRemovedLatch.countDown(); + return res; + } + + @Override + public CallbackAction segmentViewInitialized() + { + CallbackAction res = callback.segmentViewInitialized(); + segmentViewInitLatch.countDown(); + return res; + } + } + ); + } + }; + + overlordServerView = new QueryableCoordinatorServerView( + EasyMock.createMock(QueryToolChestWarehouse.class), + EasyMock.createMock(QueryWatcher.class), + getSmileMapper(), + EasyMock.createMock(HttpClient.class), + baseView, + new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), + new NoopServiceEmitter(), + new CoordinatorSegmentWatcherConfig() + ); + + baseView.start(); + overlordServerView.start(); + } + + private ObjectMapper getSmileMapper() + { + final SmileFactory smileFactory = new SmileFactory(); + smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false); + smileFactory.delegateToTextual(true); + final ObjectMapper retVal = new DefaultObjectMapper(smileFactory, "broker"); + retVal.getFactory().setCodec(retVal); + return retVal; + } + + + private DataSegment dataSegmentWithIntervalAndVersion(String intervalStr, String version) + { + return DataSegment.builder() + .dataSource("test_overlord_server_view") + .interval(Intervals.of(intervalStr)) + .loadSpec( + ImmutableMap.of( + "type", + "local", + "path", + "somewhere" + ) + ) + .version(version) + .dimensions(ImmutableList.of()) + .metrics(ImmutableList.of()) + .shardSpec(NoneShardSpec.instance()) + .binaryVersion(9) + .size(0) + .build(); + } + + @After + public void tearDown() throws Exception + { + baseView.stop(); + tearDownServerAndCurator(); + } +} From 17c351495a7b895571227fe4f396cbc220b9bd80 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 15:17:12 +0530 Subject: [PATCH 16/36] Add test for BrokerSegmentMetadataCache --- .../BrokerSegmentMetadataCacheTest.java | 1545 +++++++++++++++++ 1 file changed, 1545 insertions(+) create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java new file mode 100644 index 000000000000..eeec64ca296f --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -0,0 +1,1545 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.schema; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.druid.client.ImmutableDruidServer; +import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.coordinator.NoopCoordinatorClient; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.java.util.metrics.StubServiceEmitter; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.DruidMetrics; +import org.apache.druid.query.GlobalTableDataSource; +import org.apache.druid.query.QueryContexts; +import org.apache.druid.query.QueryRunnerFactoryConglomerate; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; +import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; +import org.apache.druid.query.metadata.metadata.ColumnAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; +import org.apache.druid.segment.IndexBuilder; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.incremental.IncrementalIndexSchema; +import org.apache.druid.segment.join.JoinConditionAnalysis; +import org.apache.druid.segment.join.Joinable; +import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.DataSourceInformation; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.TestTimelineServerView; +import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.QueryLifecycle; +import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.QueryResponse; +import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.server.security.Access; +import org.apache.druid.server.security.AllowAllAuthenticator; +import org.apache.druid.server.security.NoopEscalator; +import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.sql.calcite.table.DruidTable; +import org.apache.druid.sql.calcite.util.CalciteTestBase; +import org.apache.druid.sql.calcite.util.CalciteTests; +import org.apache.druid.sql.calcite.util.TestDataBuilder; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.partition.LinearShardSpec; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.easymock.EasyMock; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +// test polling from coordinator and when coordinator doesn't return anything +// test the result +public class BrokerSegmentMetadataCacheTest extends CalciteTestBase +{ + private final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); + // Timeout to allow (rapid) debugging, while not blocking tests with errors. + private static final int WAIT_TIMEOUT_SECS = 6; + + private SpecificSegmentsQuerySegmentWalker walker; + private TestTimelineServerView serverView; + private List druidServers; + private BrokerSegmentMetadataCache runningSchema; + private CountDownLatch buildTableLatch = new CountDownLatch(1); + private CountDownLatch markDataSourceLatch = new CountDownLatch(1); + private CountDownLatch refreshLatch = new CountDownLatch(1); + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + SegmentManager segmentManager; + Set segmentDataSourceNames; + Set joinableDataSourceNames; + JoinableFactory globalTableJoinable; + CountDownLatch getDatasourcesLatch = new CountDownLatch(1); + private QueryRunnerFactoryConglomerate conglomerate; + private Closer resourceCloser; + + private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); + static final List ROWS1 = ImmutableList.of( + TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), + TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-02", "m1", "2.0", "dim1", "10.1")), + TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-03", "m1", "3.0", "dim1", "2")) + ); + + static final List ROWS2 = ImmutableList.of( + TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-01", "m1", "4.0", "dim2", ImmutableList.of("a"))), + TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-02", "m1", "5.0", "dim2", ImmutableList.of("abc"))), + TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-03", "m1", "6.0")) + ); + + @Before + public void setUp() throws Exception + { + segmentDataSourceNames = Sets.newConcurrentHashSet(); + joinableDataSourceNames = Sets.newConcurrentHashSet(); + + segmentManager = new SegmentManager(EasyMock.createMock(SegmentLoader.class)) + { + @Override + public Set getDataSourceNames() + { + getDatasourcesLatch.countDown(); + return segmentDataSourceNames; + } + }; + + globalTableJoinable = new JoinableFactory() + { + @Override + public boolean isDirectlyJoinable(DataSource dataSource) + { + return dataSource instanceof GlobalTableDataSource && + joinableDataSourceNames.contains(((GlobalTableDataSource) dataSource).getName()); + } + + @Override + public Optional build( + DataSource dataSource, + JoinConditionAnalysis condition + ) + { + return Optional.empty(); + } + }; + + resourceCloser = Closer.create(); + conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); + + final File tmpDir = temporaryFolder.newFolder(); + final QueryableIndex index1 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(ROWS1) + .buildMMappedIndex(); + + final QueryableIndex index2 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "2")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics(new LongSumAggregatorFactory("m1", "m1")) + .withRollup(false) + .build() + ) + .rows(ROWS2) + .buildMMappedIndex(); + + final InputRowSchema rowSchema = new InputRowSchema( + new TimestampSpec("t", null, null), + DimensionsSpec.builder().useSchemaDiscovery(true).build(), + null + ); + final List autoRows1 = ImmutableList.of( + TestDataBuilder.createRow( + ImmutableMap.builder() + .put("t", "2023-01-01T00:00Z") + .put("numbery", 1.1f) + .put("numberyArrays", ImmutableList.of(1L, 2L, 3L)) + .put("stringy", ImmutableList.of("a", "b", "c")) + .put("array", ImmutableList.of(1.1, 2.2, 3.3)) + .put("nested", ImmutableMap.of("x", 1L, "y", 2L)) + .build(), + rowSchema + ) + ); + final List autoRows2 = ImmutableList.of( + TestDataBuilder.createRow( + ImmutableMap.builder() + .put("t", "2023-01-02T00:00Z") + .put("numbery", 1L) + .put("numberyArrays", ImmutableList.of(3.3, 2.2, 3.1)) + .put("stringy", "a") + .put("array", ImmutableList.of(1L, 2L, 3L)) + .put("nested", "hello") + .build(), + rowSchema + ) + ); + + final QueryableIndex indexAuto1 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec(rowSchema.getTimestampSpec()) + .withDimensionsSpec(rowSchema.getDimensionsSpec()) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows1) + .buildMMappedIndex(); + + final QueryableIndex indexAuto2 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec( + new TimestampSpec("t", null, null) + ) + .withDimensionsSpec( + DimensionsSpec.builder().useSchemaDiscovery(true).build() + ) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows2) + .buildMMappedIndex(); + + walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( + DataSegment.builder() + .dataSource(CalciteTests.DATASOURCE1) + .interval(Intervals.of("2000/P1Y")) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index1 + ).add( + DataSegment.builder() + .dataSource(CalciteTests.DATASOURCE1) + .interval(Intervals.of("2001/P1Y")) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index2 + ).add( + DataSegment.builder() + .dataSource(CalciteTests.DATASOURCE2) + .interval(index2.getDataInterval()) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index2 + ).add( + DataSegment.builder() + .dataSource(CalciteTests.SOME_DATASOURCE) + .interval(Intervals.of("2023-01-01T00Z/P1D")) + .version("1") + .shardSpec(new LinearShardSpec(1)) + .size(0) + .build(), + indexAuto1 + ).add( + DataSegment.builder() + .dataSource(CalciteTests.SOME_DATASOURCE) + .interval(Intervals.of("2023-01-02T00Z/P1D")) + .version("1") + .shardSpec(new LinearShardSpec(1)) + .size(0) + .build(), + indexAuto2 + ); + final DataSegment segment1 = new DataSegment( + "foo3", + Intervals.of("2012/2013"), + "version3", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(2, 3), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + final List realtimeSegments = ImmutableList.of(segment1); + serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); + druidServers = serverView.getDruidServers(); + } + + public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException + { + return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); + } + + public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMetadataCacheConfig config) throws InterruptedException + { + Preconditions.checkState(runningSchema == null); + runningSchema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + config, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public DataSourceInformation buildDruidTable(String dataSource) + { + DataSourceInformation table = super.buildDruidTable(dataSource); + buildTableLatch.countDown(); + return table; + } + + @Override + public void markDataSourceAsNeedRebuild(String datasource) + { + super.markDataSourceAsNeedRebuild(datasource); + markDataSourceLatch.countDown(); + } + }; + + runningSchema.start(); + runningSchema.awaitInitialization(); + return runningSchema; + } + + public BrokerSegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedException + { + Preconditions.checkState(runningSchema == null); + runningSchema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void markDataSourceAsNeedRebuild(String datasource) + { + super.markDataSourceAsNeedRebuild(datasource); + markDataSourceLatch.countDown(); + } + + @Override + @VisibleForTesting + public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws + IOException + { + super.refresh(segmentsToRefresh, dataSourcesToRebuild); + refreshLatch.countDown(); + } + }; + + runningSchema.start(); + runningSchema.awaitInitialization(); + return runningSchema; + } + + @After + public void tearDown() throws Exception + { + if (runningSchema != null) { + runningSchema.stop(); + } + walker.close(); + resourceCloser.close(); + } + + @Test + public void testGetTableMap() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema.getDatasourceNames()); + + final Set tableNames = schema.getDatasourceNames(); + Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), tableNames); + } + + @Test + public void testSchemaInit() throws InterruptedException + { + BrokerSegmentMetadataCache schema2 = buildSchemaMarkAndTableLatch(); + Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema2.getDatasourceNames()); + } + + @Test + public void testGetTableMapFoo() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata("foo"); + final DruidTable fooTable = new DatasourceTable(fooDs); + final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); + final List fields = rowType.getFieldList(); + + Assert.assertEquals(6, fields.size()); + + Assert.assertEquals("__time", fields.get(0).getName()); + Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); + + Assert.assertEquals("dim2", fields.get(1).getName()); + Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(1).getType().getSqlTypeName()); + + Assert.assertEquals("m1", fields.get(2).getName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getSqlTypeName()); + + Assert.assertEquals("dim1", fields.get(3).getName()); + Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getSqlTypeName()); + + Assert.assertEquals("cnt", fields.get(4).getName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(4).getType().getSqlTypeName()); + + Assert.assertEquals("unique_dim1", fields.get(5).getName()); + Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); + } + + @Test + public void testGetTableMapFoo2() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata("foo2"); + final DruidTable fooTable = new DatasourceTable(fooDs); + final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); + final List fields = rowType.getFieldList(); + + Assert.assertEquals(3, fields.size()); + + Assert.assertEquals("__time", fields.get(0).getName()); + Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); + + Assert.assertEquals("dim2", fields.get(1).getName()); + Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(1).getType().getSqlTypeName()); + + Assert.assertEquals("m1", fields.get(2).getName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(2).getType().getSqlTypeName()); + } + + @Test + public void testGetTableMapSomeTable() throws InterruptedException + { + // using 'newest first' column type merge strategy, the types are expected to be the types defined in the newer + // segment, except for json, which is special handled + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch( + new BrokerSegmentMetadataCacheConfig() { + @Override + public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() + { + return new SegmentMetadataCache.FirstTypeMergePolicy(); + } + } + ); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata(CalciteTests.SOME_DATASOURCE); + final DruidTable table = new DatasourceTable(fooDs); + final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); + final List fields = rowType.getFieldList(); + + Assert.assertEquals(9, fields.size()); + + Assert.assertEquals("__time", fields.get(0).getName()); + Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); + + Assert.assertEquals("numbery", fields.get(1).getName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(1).getType().getSqlTypeName()); + + Assert.assertEquals("numberyArrays", fields.get(2).getName()); + Assert.assertEquals(SqlTypeName.ARRAY, fields.get(2).getType().getSqlTypeName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getComponentType().getSqlTypeName()); + + Assert.assertEquals("stringy", fields.get(3).getName()); + Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getSqlTypeName()); + + Assert.assertEquals("array", fields.get(4).getName()); + Assert.assertEquals(SqlTypeName.ARRAY, fields.get(4).getType().getSqlTypeName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(4).getType().getComponentType().getSqlTypeName()); + + Assert.assertEquals("nested", fields.get(5).getName()); + Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); + + Assert.assertEquals("cnt", fields.get(6).getName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(6).getType().getSqlTypeName()); + + Assert.assertEquals("m1", fields.get(7).getName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(7).getType().getSqlTypeName()); + + Assert.assertEquals("unique_dim1", fields.get(8).getName()); + Assert.assertEquals(SqlTypeName.OTHER, fields.get(8).getType().getSqlTypeName()); + } + + @Test + public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws InterruptedException + { + // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the + // least restrictive blend across all segments + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata(CalciteTests.SOME_DATASOURCE); + final DruidTable table = new DatasourceTable(fooDs); + final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); + final List fields = rowType.getFieldList(); + + Assert.assertEquals(9, fields.size()); + + Assert.assertEquals("__time", fields.get(0).getName()); + Assert.assertEquals(SqlTypeName.TIMESTAMP, fields.get(0).getType().getSqlTypeName()); + + Assert.assertEquals("numbery", fields.get(1).getName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(1).getType().getSqlTypeName()); + + Assert.assertEquals("numberyArrays", fields.get(2).getName()); + Assert.assertEquals(SqlTypeName.ARRAY, fields.get(2).getType().getSqlTypeName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(2).getType().getComponentType().getSqlTypeName()); + + Assert.assertEquals("stringy", fields.get(3).getName()); + Assert.assertEquals(SqlTypeName.ARRAY, fields.get(3).getType().getSqlTypeName()); + Assert.assertEquals(SqlTypeName.VARCHAR, fields.get(3).getType().getComponentType().getSqlTypeName()); + + Assert.assertEquals("array", fields.get(4).getName()); + Assert.assertEquals(SqlTypeName.ARRAY, fields.get(4).getType().getSqlTypeName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(4).getType().getComponentType().getSqlTypeName()); + + Assert.assertEquals("nested", fields.get(5).getName()); + Assert.assertEquals(SqlTypeName.OTHER, fields.get(5).getType().getSqlTypeName()); + + Assert.assertEquals("cnt", fields.get(6).getName()); + Assert.assertEquals(SqlTypeName.BIGINT, fields.get(6).getType().getSqlTypeName()); + + Assert.assertEquals("m1", fields.get(7).getName()); + Assert.assertEquals(SqlTypeName.DOUBLE, fields.get(7).getType().getSqlTypeName()); + + Assert.assertEquals("unique_dim1", fields.get(8).getName()); + Assert.assertEquals(SqlTypeName.OTHER, fields.get(8).getType().getSqlTypeName()); + } + + /** + * This tests that {@link AvailableSegmentMetadata#getNumRows()} is correct in case + * of multiple replicas i.e. when {@link SegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} + * is called more than once for same segment + * @throws InterruptedException + */ + @Test + public void testAvailableSegmentMetadataNumRows() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); + final List segments = segmentsMetadata.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // find the only segment with datasource "foo2" + final DataSegment existingSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(existingSegment); + final AvailableSegmentMetadata existingMetadata = segmentsMetadata.get(existingSegment.getId()); + // update AvailableSegmentMetadata of existingSegment with numRows=5 + AvailableSegmentMetadata updatedMetadata = AvailableSegmentMetadata.from(existingMetadata).withNumRows(5).build(); + schema.setAvailableSegmentMetadata(existingSegment.getId(), updatedMetadata); + // find a druidServer holding existingSegment + final Pair pair = druidServers + .stream() + .flatMap(druidServer -> druidServer + .iterateAllSegments() + .stream() + .filter(segment -> segment.getId().equals(existingSegment.getId())) + .map(segment -> Pair.of(druidServer, segment)) + ) + .findAny() + .orElse(null); + Assert.assertNotNull(pair); + final ImmutableDruidServer server = pair.lhs; + Assert.assertNotNull(server); + final DruidServerMetadata druidServerMetadata = server.getMetadata(); + // invoke SegmentMetadataCache#addSegment on existingSegment + schema.addSegment(druidServerMetadata, existingSegment); + segmentsMetadata = schema.getSegmentMetadataSnapshot(); + // get the only segment with datasource "foo2" + final DataSegment currentSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + final AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); + Assert.assertEquals(updatedMetadata.getSegment().getId(), currentMetadata.getSegment().getId()); + Assert.assertEquals(updatedMetadata.getNumRows(), currentMetadata.getNumRows()); + // numreplicas do not change here since we addSegment with the same server which was serving existingSegment before + Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); + } + + @Test + public void testNullDatasource() throws IOException, InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // segments contains two segments with datasource "foo" and one with datasource "foo2" + // let's remove the only segment with datasource "foo2" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + } + + @Test + public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // remove one of the segments with datasource "foo" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + } + + @Test + public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); + final List segments = segmentsMetadata.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + // find the only realtime segment with datasource "foo3" + final DataSegment existingSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo3")) + .findFirst() + .orElse(null); + Assert.assertNotNull(existingSegment); + final AvailableSegmentMetadata metadata = segmentsMetadata.get(existingSegment.getId()); + Assert.assertEquals(1L, metadata.isRealtime()); + // get the historical server + final ImmutableDruidServer historicalServer = druidServers.stream() + .filter(s -> s.getType().equals(ServerType.HISTORICAL)) + .findAny() + .orElse(null); + + Assert.assertNotNull(historicalServer); + final DruidServerMetadata historicalServerMetadata = historicalServer.getMetadata(); + + // add existingSegment to historical + schema.addSegment(historicalServerMetadata, existingSegment); + segmentsMetadata = schema.getSegmentMetadataSnapshot(); + // get the segment with datasource "foo3" + DataSegment currentSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo3")) + .findFirst() + .orElse(null); + Assert.assertNotNull(currentSegment); + AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); + Assert.assertEquals(0L, currentMetadata.isRealtime()); + + ImmutableDruidServer realtimeServer = druidServers.stream() + .filter(s -> s.getType().equals(ServerType.REALTIME)) + .findAny() + .orElse(null); + Assert.assertNotNull(realtimeServer); + // drop existingSegment from realtime task + schema.removeServerSegment(realtimeServer.getMetadata(), existingSegment); + segmentsMetadata = schema.getSegmentMetadataSnapshot(); + currentSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo3")) + .findFirst() + .orElse(null); + Assert.assertNotNull(currentSegment); + currentMetadata = segmentsMetadata.get(currentSegment.getId()); + Assert.assertEquals(0L, currentMetadata.isRealtime()); + } + + @Test + public void testSegmentAddedCallbackAddNewHistoricalSegment() throws InterruptedException + { + String datasource = "newSegmentAddTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + SegmentMetadataCache schema = new SegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + }; + + serverView.addSegment(newSegment(datasource, 1), ServerType.HISTORICAL); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(1, metadatas.size()); + AvailableSegmentMetadata metadata = metadatas.get(0); + Assert.assertEquals(0, metadata.isRealtime()); + Assert.assertEquals(0, metadata.getNumRows()); + Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); + } + + @Test + public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedException + { + String datasource = "newSegmentAddTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(2); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + }; + + DataSegment segment = newSegment(datasource, 1); + serverView.addSegment(segment, ServerType.REALTIME); + serverView.addSegment(segment, ServerType.HISTORICAL); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(1, metadatas.size()); + AvailableSegmentMetadata metadata = metadatas.get(0); + Assert.assertEquals(0, metadata.isRealtime()); // realtime flag is unset when there is any historical + Assert.assertEquals(0, metadata.getNumRows()); + Assert.assertEquals(2, metadata.getNumReplicas()); + Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); + Assert.assertFalse(schema.getMutableSegments().contains(metadata.getSegment().getId())); + } + + @Test + public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedException + { + String datasource = "newSegmentAddTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + }; + + serverView.addSegment(newSegment(datasource, 1), ServerType.REALTIME); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(1, metadatas.size()); + AvailableSegmentMetadata metadata = metadatas.get(0); + Assert.assertEquals(1, metadata.isRealtime()); + Assert.assertEquals(0, metadata.getNumRows()); + Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); + Assert.assertTrue(schema.getMutableSegments().contains(metadata.getSegment().getId())); + } + + @Test + public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedException + { + String datasource = "newSegmentAddTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + }; + + serverView.addSegment(newSegment(datasource, 1), ServerType.BROKER); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(6, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(0, metadatas.size()); + Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); + } + + @Test + public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws InterruptedException, IOException + { + String datasource = "segmentRemoveTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + CountDownLatch removeSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + + @Override + public void removeSegment(final DataSegment segment) + { + super.removeSegment(segment); + if (datasource.equals(segment.getDataSource())) { + removeSegmentLatch.countDown(); + } + } + }; + + DataSegment segment = newSegment(datasource, 1); + serverView.addSegment(segment, ServerType.REALTIME); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + schema.refresh(Sets.newHashSet(segment.getId()), Sets.newHashSet(datasource)); + + serverView.removeSegment(segment, ServerType.REALTIME); + Assert.assertTrue(removeSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(6, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(0, metadatas.size()); + Assert.assertFalse(schema.getSegmentsNeedingRefresh().contains(segment.getId())); + Assert.assertFalse(schema.getMutableSegments().contains(segment.getId())); + Assert.assertFalse(schema.getDataSourcesNeedingRebuild().contains(datasource)); + Assert.assertFalse(schema.getDatasourceNames().contains(datasource)); + } + + @Test + public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws InterruptedException, IOException + { + String datasource = "segmentRemoveTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(2); + CountDownLatch removeSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + + @Override + public void removeSegment(final DataSegment segment) + { + super.removeSegment(segment); + if (datasource.equals(segment.getDataSource())) { + removeSegmentLatch.countDown(); + } + } + }; + + List segments = ImmutableList.of( + newSegment(datasource, 1), + newSegment(datasource, 2) + ); + serverView.addSegment(segments.get(0), ServerType.REALTIME); + serverView.addSegment(segments.get(1), ServerType.HISTORICAL); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(datasource)); + + serverView.removeSegment(segments.get(0), ServerType.REALTIME); + Assert.assertTrue(removeSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(1, metadatas.size()); + Assert.assertFalse(schema.getSegmentsNeedingRefresh().contains(segments.get(0).getId())); + Assert.assertFalse(schema.getMutableSegments().contains(segments.get(0).getId())); + Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); + Assert.assertTrue(schema.getDatasourceNames().contains(datasource)); + } + + @Test + public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws InterruptedException + { + String datasource = "serverSegmentRemoveTest"; + CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.removeServerSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + removeServerSegmentLatch.countDown(); + } + } + }; + + serverView.addSegment(newSegment(datasource, 1), ServerType.BROKER); + + serverView.removeSegment(newSegment(datasource, 1), ServerType.HISTORICAL); + Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(6, schema.getTotalSegments()); + } + + @Test + public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws InterruptedException + { + String datasource = "serverSegmentRemoveTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + + @Override + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.removeServerSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + removeServerSegmentLatch.countDown(); + } + } + }; + + DataSegment segment = newSegment(datasource, 1); + serverView.addSegment(segment, ServerType.HISTORICAL); + serverView.addSegment(segment, ServerType.BROKER); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + serverView.removeSegment(segment, ServerType.BROKER); + Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); + } + + @Test + public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws InterruptedException + { + String datasource = "serverSegmentRemoveTest"; + CountDownLatch addSegmentLatch = new CountDownLatch(1); + CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + + @Override + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.removeServerSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + removeServerSegmentLatch.countDown(); + } + } + }; + + DataSegment segment = newSegment(datasource, 1); + serverView.addSegment(segment, ServerType.HISTORICAL); + serverView.addSegment(segment, ServerType.BROKER); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + + serverView.removeSegment(segment, ServerType.HISTORICAL); + Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); + + Assert.assertEquals(7, schema.getTotalSegments()); + List metadatas = schema + .getSegmentMetadataSnapshot() + .values() + .stream() + .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) + .collect(Collectors.toList()); + Assert.assertEquals(1, metadatas.size()); + AvailableSegmentMetadata metadata = metadatas.get(0); + Assert.assertEquals(0, metadata.isRealtime()); + Assert.assertEquals(0, metadata.getNumRows()); + Assert.assertEquals(0, metadata.getNumReplicas()); // brokers are not counted as replicas yet + } + + /** + * Test actions on the cache. The current design of the cache makes testing far harder + * than it should be. + * + * - The cache is refreshed on a schedule. + * - Datasources are added to the refresh queue via an unsynchronized thread. + * - The refresh loop always refreshes since one of the segments is dynamic. + * + * The use of latches tries to keep things synchronized, but there are many + * moving parts. A simpler technique is sorely needed. + */ + @Test + public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws InterruptedException + { + BrokerSegmentMetadataCache schema3 = buildSchemaMarkAndRefreshLatch(); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + DatasourceTable.PhysicalDatasourceMetadata fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + + markDataSourceLatch = new CountDownLatch(1); + refreshLatch = new CountDownLatch(1); + final DataSegment someNewBrokerSegment = new DataSegment( + "foo", + Intervals.of("2012/2013"), + "version1", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(2, 3), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); + Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for build twice + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) + refreshLatch = new CountDownLatch(1); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + + fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + + // now remove it + markDataSourceLatch = new CountDownLatch(1); + refreshLatch = new CountDownLatch(1); + serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); + + Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for build twice + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) + refreshLatch = new CountDownLatch(1); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + + fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + } + + @Test + public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throws InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + + markDataSourceLatch = new CountDownLatch(1); + refreshLatch = new CountDownLatch(1); + final DataSegment someNewBrokerSegment = new DataSegment( + "foo", + Intervals.of("2012/2013"), + "version1", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(2, 3), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); + + Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for build twice + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) + refreshLatch = new CountDownLatch(1); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + + fooTable = schema.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + + // now remove it + markDataSourceLatch = new CountDownLatch(1); + refreshLatch = new CountDownLatch(1); + serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); + + Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for build twice + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) + refreshLatch = new CountDownLatch(1); + Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); + + fooTable = schema.getPhysicalDatasourceMetadata("foo"); + Assert.assertNotNull(fooTable); + } + + /** + * Ensure that the BrokerInternalQueryConfig context is honored for this internally generated SegmentMetadata Query + */ + @Test + public void testRunSegmentMetadataQueryWithContext() throws Exception + { + Map queryContext = ImmutableMap.of( + QueryContexts.PRIORITY_KEY, 5, + QueryContexts.BROKER_PARALLEL_MERGE_KEY, false + ); + + String brokerInternalQueryConfigJson = "{\"context\": { \"priority\": 5} }"; + + TestHelper.makeJsonMapper(); + InternalQueryConfig internalQueryConfig = MAPPER.readValue( + MAPPER.writeValueAsString( + MAPPER.readValue(brokerInternalQueryConfigJson, InternalQueryConfig.class) + ), + InternalQueryConfig.class + ); + + DataSegment segment = newSegment("test", 0); + List segmentIterable = ImmutableList.of(segment.getId()); + + // This is the query that we expect this method to create. We will be testing that it matches the query generated by the method under test. + SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( + new TableDataSource(segment.getDataSource()), + new MultipleSpecificSegmentSpec( + segmentIterable.stream() + .map(SegmentId::toDescriptor).collect(Collectors.toList())), + new AllColumnIncluderator(), + false, + queryContext, + EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), + false, + null, + null + ); + + QueryLifecycleFactory factoryMock = EasyMock.createMock(QueryLifecycleFactory.class); + QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); + + // Need to create schema for this test because the available schemas don't mock the QueryLifecycleFactory, which I need for this test. + BrokerSegmentMetadataCache mySchema = new BrokerSegmentMetadataCache( + factoryMock, + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + internalQueryConfig, + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ); + + EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); + // This is the mat of the test, making sure that the query created by the method under test matches the expected query, specifically the operator configured context + EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) + .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); + + EasyMock.replay(factoryMock, lifecycleMock); + + mySchema.runSegmentMetadataQuery(segmentIterable); + + EasyMock.verify(factoryMock, lifecycleMock); + + } + + @Test + public void testSegmentMetadataColumnType() + { + // Verify order is preserved. + final LinkedHashMap columns = new LinkedHashMap<>(); + columns.put( + "a", + new ColumnAnalysis(ColumnType.STRING, ColumnType.STRING.asTypeString(), false, true, 1234, 26, "a", "z", null) + ); + + columns.put( + "count", + new ColumnAnalysis(ColumnType.LONG, ColumnType.LONG.asTypeString(), false, true, 1234, 26, "a", "z", null) + ); + + columns.put( + "b", + new ColumnAnalysis(ColumnType.DOUBLE, ColumnType.DOUBLE.asTypeString(), false, true, 1234, 26, null, null, null) + ); + + RowSignature signature = SegmentMetadataCache.analysisToRowSignature( + new SegmentAnalysis( + "id", + ImmutableList.of(Intervals.utc(1L, 2L)), + columns, + 1234, + 100, + null, + null, + null, + null + ) + ); + + Assert.assertEquals( + RowSignature.builder() + .add("a", ColumnType.STRING) + .add("count", ColumnType.LONG) + .add("b", ColumnType.DOUBLE) + .build(), + signature + ); + } + + @Test + public void testSegmentMetadataFallbackType() + { + RowSignature signature = SegmentMetadataCache.analysisToRowSignature( + new SegmentAnalysis( + "id", + ImmutableList.of(Intervals.utc(1L, 2L)), + new LinkedHashMap<>( + ImmutableMap.of( + "a", + new ColumnAnalysis( + null, + ColumnType.STRING.asTypeString(), + false, + true, + 1234, + 26, + "a", + "z", + null + ), + "count", + new ColumnAnalysis( + null, + ColumnType.LONG.asTypeString(), + false, + true, + 1234, + 26, + "a", + "z", + null + ) + ) + ), + 1234, + 100, + null, + null, + null, + null + ) + ); + Assert.assertEquals( + RowSignature.builder().add("a", ColumnType.STRING).add("count", ColumnType.LONG).build(), + signature + ); + } + + @Test + public void testStaleDatasourceRefresh() throws IOException, InterruptedException + { + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Set segments = new HashSet<>(); + Set datasources = new HashSet<>(); + datasources.add("wat"); + Assert.assertNull(schema.getDatasource("wat")); + schema.refresh(segments, datasources); + Assert.assertNull(schema.getDatasource("wat")); + } + + @Test + public void testRefreshShouldEmitMetrics() throws InterruptedException, IOException + { + String datasource = "xyz"; + CountDownLatch addSegmentLatch = new CountDownLatch(2); + StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + emitter, + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new NoopCoordinatorClient() + ) + { + @Override + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + super.addSegment(server, segment); + if (datasource.equals(segment.getDataSource())) { + addSegmentLatch.countDown(); + } + } + + @Override + public void removeSegment(final DataSegment segment) + { + super.removeSegment(segment); + } + }; + + List segments = ImmutableList.of( + newSegment(datasource, 1), + newSegment(datasource, 2) + ); + serverView.addSegment(segments.get(0), ServerType.HISTORICAL); + serverView.addSegment(segments.get(1), ServerType.REALTIME); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(datasource)); + + emitter.verifyEmitted("metadatacache/refresh/time", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); + emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); + } + + private static DataSegment newSegment(String datasource, int partitionId) + { + return new DataSegment( + datasource, + Intervals.of("2012/2013"), + "version1", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(partitionId, 0), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + } +} From f0baf33d55896155f7543d8f12242593d24b7e29 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 15:17:30 +0530 Subject: [PATCH 17/36] minor code changes and fix checkstyle issues --- ...ruidSchemaInternRowSignatureBenchmark.java | 9 ++--- .../druid/benchmark/query/SqlBenchmark.java | 2 +- .../query/SqlExpressionBenchmark.java | 2 +- .../query/SqlNestedDataBenchmark.java | 2 +- .../benchmark/query/SqlVsNativeBenchmark.java | 2 +- ...ressedBigDecimalSqlAggregatorTestBase.java | 2 +- .../sql/TDigestSketchSqlAggregatorTest.java | 2 +- .../hll/sql/HllSketchSqlAggregatorTest.java | 2 +- .../sql/DoublesSketchSqlAggregatorTest.java | 2 +- .../sql/ThetaSketchSqlAggregatorTest.java | 2 +- ...ArrayOfDoublesSketchSqlAggregatorTest.java | 2 +- .../sql/BloomFilterSqlAggregatorTest.java | 2 +- ...etsHistogramQuantileSqlAggregatorTest.java | 2 +- .../sql/QuantileSqlAggregatorTest.java | 2 +- .../apache/druid/msq/test/MSQTestBase.java | 2 +- .../sql/VarianceSqlAggregatorTest.java | 2 +- .../apache/druid/client/BrokerServerView.java | 2 -- .../coordinator/CoordinatorClientImpl.java | 3 +- .../metadata/DataSourceInformation.java | 19 ++++++++++ .../metadata/SegmentMetadataCache.java | 18 +++++----- .../metadata/SegmentMetadataCacheConfig.java | 8 +++++ .../server/http/DataSourcesResource.java | 2 +- .../SegmentDataCacheConcurrencyTest.java | 2 +- .../metadata/SegmentMetadataCacheCommon.java | 4 +-- .../SegmentMetadataCacheConfigTest.java | 2 -- .../metadata/SegmentMetadataCacheTest.java | 35 +++++++++---------- .../server/http/MetadataResourceTest.java | 13 +++---- .../java/org/apache/druid/cli/CliBroker.java | 6 ++-- .../org/apache/druid/cli/CliCoordinator.java | 2 +- .../schema/BrokerSegmentMetadataCache.java | 3 +- .../BrokerSegmentMetadataCacheConfig.java | 10 ++++++ .../schema/DruidCalciteSchemaModule.java | 2 -- .../druid/sql/calcite/schema/DruidSchema.java | 2 -- .../apache/druid/sql/SqlStatementTest.java | 2 +- .../sql/avatica/DruidAvaticaHandlerTest.java | 2 +- .../druid/sql/avatica/DruidStatementTest.java | 2 +- .../sql/calcite/BaseCalciteQueryTest.java | 2 +- .../calcite/CalciteNestedDataQueryTest.java | 2 +- .../SqlVectorizedExpressionSanityTest.java | 2 +- .../schema/DruidSchemaNoDataInitTest.java | 5 ++- ...PhysicalDataSourceMetadataBuilderTest.java | 3 +- .../sql/calcite/schema/SystemSchemaTest.java | 1 - .../sql/calcite/util/QueryFrameworkUtils.java | 4 +-- .../sql/calcite/util/SqlTestFramework.java | 2 +- .../sql/calcite/util/TestDataBuilder.java | 2 +- .../druid/sql/http/SqlResourceTest.java | 2 +- 46 files changed, 112 insertions(+), 91 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 4d2d11ba1f3f..5f0634f283a4 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -30,16 +30,13 @@ import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.join.JoinableFactory; +import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.SegmentManager; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Escalator; -import org.apache.druid.sql.calcite.planner.PlannerConfig; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.partition.LinearShardSpec; @@ -104,7 +101,7 @@ public void addSegment(final DruidServerMetadata server, final DataSegment segme } @Override - protected Sequence runSegmentMetadataQuery(Iterable segments) + public Sequence runSegmentMetadataQuery(Iterable segments) { final int numColumns = 1000; LinkedHashMap columnToAnalysisMap = new LinkedHashMap<>(); diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java index d340f10e35e3..9bfe925ccf32 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlBenchmark.java @@ -46,6 +46,7 @@ import org.apache.druid.segment.generator.GeneratorSchemaInfo; import org.apache.druid.segment.generator.SegmentGenerator; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.sql.calcite.aggregation.ApproxCountDistinctSqlAggregator; @@ -63,7 +64,6 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java index 15c26f1e4577..dbdd4ea19620 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlExpressionBenchmark.java @@ -36,6 +36,7 @@ import org.apache.druid.segment.generator.GeneratorSchemaInfo; import org.apache.druid.segment.generator.SegmentGenerator; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.sql.calcite.SqlVectorizedExpressionSanityTest; @@ -48,7 +49,6 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java index 7fc8a71c3fd3..1915776dbca3 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java @@ -45,6 +45,7 @@ import org.apache.druid.segment.transform.ExpressionTransform; import org.apache.druid.segment.transform.TransformSpec; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.sql.calcite.SqlVectorizedExpressionSanityTest; @@ -57,7 +58,6 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java index 8c709a7457f9..e4f8b5570bf0 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlVsNativeBenchmark.java @@ -38,6 +38,7 @@ import org.apache.druid.segment.generator.GeneratorSchemaInfo; import org.apache.druid.segment.generator.SegmentGenerator; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.sql.calcite.planner.CalciteRulesManager; @@ -49,7 +50,6 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.openjdk.jmh.annotations.Benchmark; diff --git a/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java b/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java index b6aa0d866e5d..cda55e314e0b 100644 --- a/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java +++ b/extensions-contrib/compressed-bigdecimal/src/test/java/org/apache/druid/compressedbigdecimal/CompressedBigDecimalSqlAggregatorTestBase.java @@ -41,10 +41,10 @@ import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java index dd15a6defac2..476b7734fca9 100644 --- a/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java +++ b/extensions-contrib/tdigestsketch/src/test/java/org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java @@ -46,10 +46,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java index c08bc8f04345..6f69237546e3 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java @@ -73,10 +73,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.DataSegment; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java index 68f53687f892..e85c49a5274b 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java @@ -57,10 +57,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java index b4d7e29661cb..d5ae2d63f8b1 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/theta/sql/ThetaSketchSqlAggregatorTest.java @@ -59,10 +59,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.DataSegment; diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java index f262accfef94..77031dbb02ab 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/sql/ArrayOfDoublesSketchSqlAggregatorTest.java @@ -46,10 +46,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java index 30b76e8602b5..93902b0996ee 100644 --- a/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java +++ b/extensions-core/druid-bloom-filter/src/test/java/org/apache/druid/query/aggregation/bloom/sql/BloomFilterSqlAggregatorTest.java @@ -47,10 +47,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java index fea3d871e938..5b529e188e10 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/FixedBucketsHistogramQuantileSqlAggregatorTest.java @@ -49,10 +49,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java index 0c81b4c40fcb..b9d4000b6f84 100644 --- a/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java +++ b/extensions-core/histogram/src/test/java/org/apache/druid/query/aggregation/histogram/sql/QuantileSqlAggregatorTest.java @@ -48,10 +48,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java index 7c1d99fea5fa..68acd4744d3c 100644 --- a/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java +++ b/extensions-core/multi-stage-query/src/test/java/org/apache/druid/msq/test/MSQTestBase.java @@ -143,6 +143,7 @@ import org.apache.druid.segment.realtime.appenderator.AppenderatorsManager; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DataSegmentAnnouncer; import org.apache.druid.server.coordination.NoopDataSegmentAnnouncer; import org.apache.druid.server.security.AuthConfig; @@ -164,7 +165,6 @@ import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.LookylooModule; import org.apache.druid.sql.calcite.util.QueryFrameworkUtils; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.SqlTestFramework; import org.apache.druid.sql.calcite.view.InProcessViewManager; import org.apache.druid.sql.guice.SqlBindings; diff --git a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java index 23d404354701..523b7128a0f7 100644 --- a/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java +++ b/extensions-core/stats/src/test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java @@ -55,10 +55,10 @@ import org.apache.druid.segment.serde.ComplexMetrics; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.BaseCalciteQueryTest; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/server/src/main/java/org/apache/druid/client/BrokerServerView.java b/server/src/main/java/org/apache/druid/client/BrokerServerView.java index 5285b243f158..77fc59219096 100644 --- a/server/src/main/java/org/apache/druid/client/BrokerServerView.java +++ b/server/src/main/java/org/apache/druid/client/BrokerServerView.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; import com.google.inject.Inject; import org.apache.druid.client.selector.QueryableDruidServer; @@ -54,7 +53,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java index f257a71de5b4..dfe3099c8cf5 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClientImpl.java @@ -114,8 +114,7 @@ public ListenableFuture> fetchUsedSegments(String dataSource, public ListenableFuture> fetchDataSourceInformation(Set dataSources) { final String path = "/druid/coordinator/v1/metadata/dataSourceInformation"; - if (null == dataSources) - { + if (null == dataSources) { dataSources = new HashSet<>(); } return FutureUtils.transform( diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index 6501704bddac..2e3257e100c1 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.apache.druid.segment.metadata; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 704fc52de5f8..e5709fac0dbd 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -470,7 +470,7 @@ public Set getDatasourceNames() } @VisibleForTesting - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { // Get lock first so that we won't wait in ConcurrentMap.compute(). synchronized (lock) { @@ -542,7 +542,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @VisibleForTesting - void removeSegment(final DataSegment segment) + public void removeSegment(final DataSegment segment) { // Get lock first so that we won't wait in ConcurrentMap.compute(). synchronized (lock) { @@ -580,7 +580,7 @@ void removeSegment(final DataSegment segment) } @VisibleForTesting - void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) { // Get lock first so that we won't wait in ConcurrentMap.compute(). synchronized (lock) { @@ -664,7 +664,7 @@ private void unmarkSegmentAsMutable(SegmentId segmentId) } @VisibleForTesting - void markDataSourceAsNeedRebuild(String datasource) + public void markDataSourceAsNeedRebuild(String datasource) { synchronized (lock) { dataSourcesNeedingRebuild.add(datasource); @@ -877,7 +877,7 @@ public int getTotalSegments() } @VisibleForTesting - TreeSet getSegmentsNeedingRefresh() + public TreeSet getSegmentsNeedingRefresh() { synchronized (lock) { return segmentsNeedingRefresh; @@ -885,7 +885,7 @@ TreeSet getSegmentsNeedingRefresh() } @VisibleForTesting - TreeSet getMutableSegments() + public TreeSet getMutableSegments() { synchronized (lock) { return mutableSegments; @@ -893,7 +893,7 @@ TreeSet getMutableSegments() } @VisibleForTesting - Set getDataSourcesNeedingRebuild() + public Set getDataSourcesNeedingRebuild() { synchronized (lock) { return dataSourcesNeedingRebuild; @@ -907,7 +907,7 @@ Set getDataSourcesNeedingRebuild() * @return {@link Sequence} of {@link SegmentAnalysis} objects */ @VisibleForTesting - protected Sequence runSegmentMetadataQuery( + public Sequence runSegmentMetadataQuery( final Iterable segments ) { @@ -979,7 +979,7 @@ public static RowSignature analysisToRowSignature(final SegmentAnalysis analysis * This method is not thread-safe and must be used only in unit tests. */ @VisibleForTesting - void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) + public void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) { final ConcurrentSkipListMap dataSourceSegments = segmentMetadataInfo .computeIfAbsent( diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index 4cf8d0fb0019..2ebcb0e53c0a 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -20,12 +20,14 @@ package org.apache.druid.segment.metadata; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; import org.joda.time.Period; public class SegmentMetadataCacheConfig { @JsonProperty private boolean awaitInitializationOnStart = false; + @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); @JsonProperty @@ -49,6 +51,12 @@ public static SegmentMetadataCacheConfig create( return config; } + @VisibleForTesting + public void setMetadataRefreshPeriod(Period metadataRefreshPeriod) + { + this.metadataRefreshPeriod = metadataRefreshPeriod; + } + public boolean isAwaitInitializationOnStart() { return awaitInitializationOnStart; diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java index 1cbaa92ae231..c8e998e72cfa 100644 --- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java +++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java @@ -30,9 +30,9 @@ import com.sun.jersey.spi.container.ResourceFilters; import it.unimi.dsi.fastutil.objects.Object2LongMap; import org.apache.commons.lang.StringUtils; +import org.apache.druid.client.CoordinatorTimeline; import org.apache.druid.client.DruidDataSource; import org.apache.druid.client.DruidServer; -import org.apache.druid.client.CoordinatorTimeline; import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.client.ImmutableSegmentLoadInfo; import org.apache.druid.client.SegmentLoadInfo; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index 60421be19810..1746d36b3e16 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -22,12 +22,12 @@ import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; -import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.BrokerServerView; import org.apache.druid.client.DruidServer; import org.apache.druid.client.FilteredServerInventoryView; import org.apache.druid.client.FilteringSegmentCallback; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.ServerView.CallbackAction; import org.apache.druid.client.ServerView.SegmentCallback; import org.apache.druid.client.ServerView.ServerRemovedCallback; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 0bd83f88f2d2..897f9f0a6463 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -42,7 +42,6 @@ import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TemporaryFolder; @@ -122,7 +121,8 @@ InputRow createRow(final ImmutableMap map, InputRowSchema inputRowSch return MapInputRowParser.parse(inputRowSchema, (Map) map); } - QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) { + QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) + { return new QueryLifecycleFactory( queryToolChestWarehouse, walker, diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java index cf0511dd6f8a..d625229972fe 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java @@ -24,8 +24,6 @@ import org.apache.druid.guice.GuiceInjectors; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.JsonConfigurator; -import org.apache.druid.segment.metadata.SegmentMetadataCache; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.joda.time.Period; import org.junit.Assert; import org.junit.Test; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index aa9df38b9b71..1237aced8254 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -296,7 +296,7 @@ public DataSourceInformation buildDruidTable(String dataSource) } @Override - void markDataSourceAsNeedRebuild(String datasource) + public void markDataSourceAsNeedRebuild(String datasource) { super.markDataSourceAsNeedRebuild(datasource); markDataSourceLatch.countDown(); @@ -408,8 +408,6 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); } - - /** * This tests that {@link AvailableSegmentMetadata#getNumRows()} is correct in case * of multiple replicas i.e. when {@link SegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} @@ -585,7 +583,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -626,7 +624,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -671,7 +669,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -713,7 +711,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -752,7 +750,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -761,7 +759,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @Override - void removeSegment(final DataSegment segment) + public void removeSegment(final DataSegment segment) { super.removeSegment(segment); if (datasource.equals(segment.getDataSource())) { @@ -808,7 +806,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -817,7 +815,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @Override - void removeSegment(final DataSegment segment) + public void removeSegment(final DataSegment segment) { super.removeSegment(segment); if (datasource.equals(segment.getDataSource())) { @@ -867,7 +865,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr ) { @Override - void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) { super.removeServerSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -900,7 +898,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -909,7 +907,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @Override - void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) { super.removeServerSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -946,7 +944,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -955,7 +953,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @Override - void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) { super.removeServerSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -1207,7 +1205,6 @@ public void testSegmentMetadataColumnType() ); } - @Test public void testSegmentMetadataFallbackType() { @@ -1285,7 +1282,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept ) { @Override - protected void addSegment(final DruidServerMetadata server, final DataSegment segment) + public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); if (datasource.equals(segment.getDataSource())) { @@ -1294,7 +1291,7 @@ protected void addSegment(final DruidServerMetadata server, final DataSegment se } @Override - void removeSegment(final DataSegment segment) + public void removeSegment(final DataSegment segment) { super.removeSegment(segment); } diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index 141c0ddc5c39..e122334946e5 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -131,10 +131,10 @@ public void testGetAllSegmentsWithOvershadowedStatus() final List resultList = extractResponseList(response); Assert.assertEquals(resultList.size(), 4); Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, null, false), resultList.get(0)); - Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, null, false), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, null, false), resultList.get(1)); Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, null, false), resultList.get(2)); // Replication factor should be 0 as the segment is overshadowed - Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); } @Test @@ -228,16 +228,17 @@ public void testGetAllSegmentsIncludingRealtime() final List resultList = extractResponseList(response); Assert.assertEquals(resultList.size(), 6); Assert.assertEquals(new SegmentStatusInCluster(segments[0], false, 2, 20L, false), resultList.get(0)); - Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 30L, false), resultList.get(1)); + Assert.assertEquals(new SegmentStatusInCluster(segments[1], false, null, 30L, false), resultList.get(1)); Assert.assertEquals(new SegmentStatusInCluster(segments[2], false, 1, null, false), resultList.get(2)); // Replication factor should be 0 as the segment is overshadowed - Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); + Assert.assertEquals(new SegmentStatusInCluster(segments[3], true, 0, null, false), resultList.get(3)); Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[0], false, null, 10L, true), resultList.get(4)); - Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[1], false, null, 40L, true), resultList.get(5)); + Assert.assertEquals(new SegmentStatusInCluster(realTimeSegments[1], false, null, 40L, true), resultList.get(5)); } @Test - public void testGetDataSourceInformation() { + public void testGetDataSourceInformation() + { SegmentMetadataCache segmentMetadataCache = Mockito.mock(SegmentMetadataCache.class); Map dataSourceInformationMap = new HashMap<>(); diff --git a/services/src/main/java/org/apache/druid/cli/CliBroker.java b/services/src/main/java/org/apache/druid/cli/CliBroker.java index 854560540664..ba8bd1234eb4 100644 --- a/services/src/main/java/org/apache/druid/cli/CliBroker.java +++ b/services/src/main/java/org/apache/druid/cli/CliBroker.java @@ -26,11 +26,11 @@ import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.name.Names; -import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.BrokerSegmentWatcherConfig; import org.apache.druid.client.BrokerServerView; import org.apache.druid.client.CachingClusteredClient; import org.apache.druid.client.HttpServerInventoryViewResource; +import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.cache.CacheConfig; import org.apache.druid.client.selector.CustomTierSelectorStrategyConfig; @@ -71,7 +71,7 @@ import org.apache.druid.server.metrics.QueryCountStatsProvider; import org.apache.druid.server.metrics.SubqueryCountStatsProvider; import org.apache.druid.server.router.TieredBrokerConfig; -import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataView; +import org.apache.druid.sql.calcite.schema.MetadataSegmentView; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.timeline.PruneLoadSpec; import org.eclipse.jetty.server.Server; @@ -129,7 +129,7 @@ protected List getModules() binder.bind(CachingClusteredClient.class).in(LazySingleton.class); LifecycleModule.register(binder, BrokerServerView.class); - LifecycleModule.register(binder, BrokerSegmentMetadataView.class); + LifecycleModule.register(binder, MetadataSegmentView.class); binder.bind(TimelineServerView.class).to(BrokerServerView.class).in(LazySingleton.class); JsonConfigProvider.bind(binder, "druid.broker.cache", CacheConfig.class); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index c61ef1c6c70d..7a3833736cdd 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -40,8 +40,8 @@ import org.apache.druid.client.CachingClusteredClient; import org.apache.druid.client.CoordinatorSegmentWatcherConfig; import org.apache.druid.client.CoordinatorServerView; -import org.apache.druid.client.HttpServerInventoryViewResource; import org.apache.druid.client.CoordinatorTimeline; +import org.apache.druid.client.HttpServerInventoryViewResource; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.QueryableCoordinatorServerView; import org.apache.druid.client.TimelineServerView; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 8a037e7cd039..8c2369aaab62 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -96,7 +96,8 @@ public void refresh(final Set segmentsToRefresh, final Set da item.getDatasource(), physicalDatasourceMetadataBuilder.build(item) )); - } catch (Exception e) { + } + catch (Exception e) { log.warn(e, "Exception querying coordinator to fetch dataSourceInformation."); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index 2554753f79c3..a51aea811e93 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; +import org.joda.time.Period; public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig { @@ -38,6 +39,15 @@ public static BrokerSegmentMetadataCacheConfig create() return new BrokerSegmentMetadataCacheConfig(); } + public static BrokerSegmentMetadataCacheConfig create( + String metadataRefreshPeriod + ) + { + BrokerSegmentMetadataCacheConfig config = new BrokerSegmentMetadataCacheConfig(); + config.setMetadataRefreshPeriod(new Period(metadataRefreshPeriod)); + return config; + } + public boolean isMetadataSegmentCacheEnable() { return metadataSegmentCacheEnable; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java index 132f2cb92590..26d038ef4a3b 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidCalciteSchemaModule.java @@ -29,8 +29,6 @@ import org.apache.druid.guice.LifecycleModule; import org.apache.druid.sql.guice.SqlBindings; -import java.util.Properties; - /** * The module responsible for providing bindings to Calcite schemas. */ diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java index aa1e7965c27e..69b96403aa82 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java @@ -20,11 +20,9 @@ package org.apache.druid.sql.calcite.schema; import org.apache.calcite.schema.Table; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.sql.calcite.table.DatasourceTable; import javax.inject.Inject; - import java.util.Set; public class DruidSchema extends AbstractTableSchema diff --git a/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java b/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java index cad98cd8409e..9b28df2c0f1b 100644 --- a/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java +++ b/sql/src/test/java/org/apache/druid/sql/SqlStatementTest.java @@ -40,6 +40,7 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.initialization.ServerConfig; import org.apache.druid.server.log.TestRequestLogger; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -58,7 +59,6 @@ import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.http.SqlQuery; import org.easymock.EasyMock; import org.hamcrest.MatcherAssert; diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index bcc2dddb4c9e..90e75e10f9ca 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -63,6 +63,7 @@ import org.apache.druid.server.QuerySchedulerProvider; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.RequestLogLine; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.initialization.ServerConfig; import org.apache.druid.server.log.RequestLogger; import org.apache.druid.server.log.TestRequestLogger; @@ -88,7 +89,6 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.guice.SqlModule; import org.eclipse.jetty.server.Server; import org.joda.time.DateTime; diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java index 1df137e772c3..c9dd67b5c7da 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidStatementTest.java @@ -32,6 +32,7 @@ import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; @@ -47,7 +48,6 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.junit.After; import org.junit.AfterClass; import org.junit.Assert; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index ecd104d0b669..0afb275a4509 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -81,6 +81,7 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.server.security.ForbiddenException; @@ -97,7 +98,6 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.util.SqlTestFramework; import org.apache.druid.sql.calcite.util.SqlTestFramework.Builder; import org.apache.druid.sql.calcite.util.SqlTestFramework.PlannerComponentSupplier; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index d366845bb9b2..e11c12313734 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -68,8 +68,8 @@ import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.virtual.NestedFieldVirtualColumn; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; -import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java index a439f1d19ea4..fc9c63cc6675 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlVectorizedExpressionSanityTest.java @@ -37,6 +37,7 @@ import org.apache.druid.segment.generator.SegmentGenerator; import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.sql.calcite.planner.CalciteRulesManager; @@ -48,7 +49,6 @@ import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.testing.InitializedNullHandlingTest; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java index 974edbd3a588..2ac84ed76de1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java @@ -26,15 +26,14 @@ import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.segment.join.MapJoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.TestTimelineServerView; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.NoopEscalator; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; -import org.apache.druid.segment.metadata.TestTimelineServerView; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java index dc670af203e4..4a8a7e33c24d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java @@ -89,7 +89,8 @@ public Optional build( } @Test - public void testBuild() { + public void testBuild() + { segmentDataSourceNames.add("foo"); joinableDataSourceNames.add("foo"); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index ed7f083dda89..4f0aaaa8e9df 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -88,7 +88,6 @@ import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.coordinator.BytesAccumulatingResponseHandler; -import org.apache.druid.server.coordinator.simulate.TestServerInventoryView; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Action; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index dc716b2a45c1..14e1912a93a9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -43,10 +43,10 @@ import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; import org.apache.druid.segment.metadata.TestTimelineServerView; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.log.NoopRequestLogger; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.AuthConfig; @@ -57,7 +57,6 @@ import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerFactory; -import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache; import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCacheConfig; @@ -79,7 +78,6 @@ import org.easymock.EasyMock; import javax.annotation.Nullable; - import java.util.ArrayList; import java.util.HashSet; import java.util.List; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 7e012b435811..d06ad3c484d6 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -40,10 +40,10 @@ import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.segment.join.JoinableFactoryWrapper; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthorizerMapper; import org.apache.druid.sql.SqlStatementFactory; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java index edfa2fc2d334..854df66aa12e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/TestDataBuilder.java @@ -61,10 +61,10 @@ import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.join.table.IndexedTableJoinable; import org.apache.druid.segment.join.table.RowBasedIndexedTable; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.LinearShardSpec; import org.apache.druid.timeline.partition.NumberedShardSpec; diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index eefbe389e62d..4a3f74b77bdd 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -70,6 +70,7 @@ import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.ResponseContextConfig; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.initialization.ServerConfig; import org.apache.druid.server.log.TestRequestLogger; import org.apache.druid.server.mocks.MockHttpServletRequest; @@ -103,7 +104,6 @@ import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.QueryLogHook; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.hamcrest.CoreMatchers; import org.hamcrest.MatcherAssert; import org.junit.After; From f18c060e152955eb6e26f600c32fc594fd792d2b Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 16:07:56 +0530 Subject: [PATCH 18/36] Fix intellij inspections --- .../java/org/apache/druid/sql/calcite/schema/SystemSchema.java | 2 +- .../calcite/schema/PhysicalDataSourceMetadataBuilderTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 48e3a4a44268..3e134d2e74fa 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -305,7 +305,7 @@ public Enumerable scan(DataContext root) final DataSegment segment = val.getDataSegment(); segmentsAlreadySeen.add(segment.getId()); final PartialSegmentData partialSegmentData = partialSegmentDataMap.get(segment.getId()); - long numReplicas = 0L, numRows = 0L, isRealtime = 0L, isAvailable = 0L; + long numReplicas = 0L, numRows = 0L, isRealtime, isAvailable = 0L; if (partialSegmentData != null) { numReplicas = partialSegmentData.getNumReplicas(); numRows = partialSegmentData.getNumRows(); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java index 4a8a7e33c24d..987a6d1e0100 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java @@ -52,7 +52,7 @@ public class PhysicalDataSourceMetadataBuilderTest private PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; @Before - public void setUp() throws Exception + public void setUp() { segmentDataSourceNames = Sets.newConcurrentHashSet(); joinableDataSourceNames = Sets.newConcurrentHashSet(); From 03383e6e06056211f237ea6cc4b631ca4954078e Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 16:08:13 +0530 Subject: [PATCH 19/36] Fix QueryableCoordinatorServerView test --- .../druid/client/QueryableCoordinatorServerViewTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java index 0bb391165feb..fccbd1651df8 100644 --- a/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java +++ b/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java @@ -142,6 +142,7 @@ public void testSingleServerAddedRemovedSegment() throws Exception unannounceSegmentForServer(druidServer, segment); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); Assert.assertEquals( 0, ((List) timeline.lookup(Intervals.of("2014-10-20T00:00:00Z/P1D"))).size() @@ -248,6 +249,8 @@ public DataSegment apply(Pair input) } Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); + timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + Assert.assertEquals( 0, ((List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09"))).size() From e5c4b39f43d27b1af831939041a0659e74163d63 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 17:52:26 +0530 Subject: [PATCH 20/36] Complete tests for SMC to verify DataSourceInformation --- .../metadata/SegmentMetadataCacheTest.java | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index 1237aced8254..f300316e6ad8 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -372,6 +372,27 @@ public void testGetTableMapFoo() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo"); + final RowSignature fooRowSignature = fooDs.getRowSignature(); + List columnNames = fooRowSignature.getColumnNames(); + Assert.assertEquals(6, columnNames.size()); + + Assert.assertEquals("__time", columnNames.get(0)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(0)).get()); + + Assert.assertEquals("dim2", columnNames.get(1)); + Assert.assertEquals(ColumnType.STRING, fooRowSignature.getColumnType(columnNames.get(1)).get()); + + Assert.assertEquals("m1", columnNames.get(2)); + Assert.assertEquals(ColumnType.DOUBLE, fooRowSignature.getColumnType(columnNames.get(2)).get()); + + Assert.assertEquals("dim1", columnNames.get(3)); + Assert.assertEquals(ColumnType.STRING, fooRowSignature.getColumnType(columnNames.get(3)).get()); + + Assert.assertEquals("cnt", columnNames.get(4)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(4)).get()); + + Assert.assertEquals("unique_dim1", columnNames.get(5)); + Assert.assertEquals(ColumnType.ofComplex("hyperUnique"), fooRowSignature.getColumnType(columnNames.get(5)).get()); } @Test @@ -379,6 +400,18 @@ public void testGetTableMapFoo2() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo2"); + final RowSignature fooRowSignature = fooDs.getRowSignature(); + List columnNames = fooRowSignature.getColumnNames(); + Assert.assertEquals(3, columnNames.size()); + + Assert.assertEquals("__time", columnNames.get(0)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(0)).get()); + + Assert.assertEquals("dim2", columnNames.get(1)); + Assert.assertEquals(ColumnType.STRING, fooRowSignature.getColumnType(columnNames.get(1)).get()); + + Assert.assertEquals("m1", columnNames.get(2)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(2)).get()); } @Test @@ -396,7 +429,36 @@ public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePoli } ); final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); + final RowSignature fooRowSignature = fooDs.getRowSignature(); + List columnNames = fooRowSignature.getColumnNames(); + Assert.assertEquals(9, columnNames.size()); + + Assert.assertEquals("__time", columnNames.get(0)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(0)).get()); + + Assert.assertEquals("numbery", columnNames.get(1)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(1)).get()); + + Assert.assertEquals("numberyArrays", columnNames.get(2)); + Assert.assertEquals(ColumnType.DOUBLE_ARRAY, fooRowSignature.getColumnType(columnNames.get(2)).get()); + + Assert.assertEquals("stringy", columnNames.get(3)); + Assert.assertEquals(ColumnType.STRING, fooRowSignature.getColumnType(columnNames.get(3)).get()); + + Assert.assertEquals("array", columnNames.get(4)); + Assert.assertEquals(ColumnType.LONG_ARRAY, fooRowSignature.getColumnType(columnNames.get(4)).get()); + Assert.assertEquals("nested", columnNames.get(5)); + Assert.assertEquals(ColumnType.ofComplex("json"), fooRowSignature.getColumnType(columnNames.get(5)).get()); + + Assert.assertEquals("cnt", columnNames.get(6)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(6)).get()); + + Assert.assertEquals("m1", columnNames.get(7)); + Assert.assertEquals(ColumnType.DOUBLE, fooRowSignature.getColumnType(columnNames.get(7)).get()); + + Assert.assertEquals("unique_dim1", columnNames.get(8)); + Assert.assertEquals(ColumnType.ofComplex("hyperUnique"), fooRowSignature.getColumnType(columnNames.get(8)).get()); } @Test @@ -406,6 +468,37 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt // least restrictive blend across all segments SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); + + final RowSignature fooRowSignature = fooDs.getRowSignature(); + List columnNames = fooRowSignature.getColumnNames(); + Assert.assertEquals(9, columnNames.size()); + + Assert.assertEquals("__time", columnNames.get(0)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(0)).get()); + + Assert.assertEquals("numbery", columnNames.get(1)); + Assert.assertEquals(ColumnType.DOUBLE, fooRowSignature.getColumnType(columnNames.get(1)).get()); + + Assert.assertEquals("numberyArrays", columnNames.get(2)); + Assert.assertEquals(ColumnType.DOUBLE_ARRAY, fooRowSignature.getColumnType(columnNames.get(2)).get()); + + Assert.assertEquals("stringy", columnNames.get(3)); + Assert.assertEquals(ColumnType.STRING_ARRAY, fooRowSignature.getColumnType(columnNames.get(3)).get()); + + Assert.assertEquals("array", columnNames.get(4)); + Assert.assertEquals(ColumnType.DOUBLE_ARRAY, fooRowSignature.getColumnType(columnNames.get(4)).get()); + + Assert.assertEquals("nested", columnNames.get(5)); + Assert.assertEquals(ColumnType.ofComplex("json"), fooRowSignature.getColumnType(columnNames.get(5)).get()); + + Assert.assertEquals("cnt", columnNames.get(6)); + Assert.assertEquals(ColumnType.LONG, fooRowSignature.getColumnType(columnNames.get(6)).get()); + + Assert.assertEquals("m1", columnNames.get(7)); + Assert.assertEquals(ColumnType.DOUBLE, fooRowSignature.getColumnType(columnNames.get(7)).get()); + + Assert.assertEquals("unique_dim1", columnNames.get(8)); + Assert.assertEquals(ColumnType.ofComplex("hyperUnique"), fooRowSignature.getColumnType(columnNames.get(8)).get()); } /** From 45378d53821a2899c2e56b9d503c1426fd786cad Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Mon, 11 Sep 2023 17:52:41 +0530 Subject: [PATCH 21/36] Add comments --- .../QueryableCoordinatorServerView.java | 36 ++++++++++++++----- .../druid/client/selector/ServerSelector.java | 6 ++++ .../metadata/DataSourceInformation.java | 2 +- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index b5f8973519f9..719562a73716 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -43,6 +43,12 @@ /** * ServerView of coordinator for the state of segments being loaded in the cluster. + *
    + * This class simply extends {@link BrokerServerView} and implements methods from {@link CoordinatorTimeline} + * for backward compatibility. This newer implementation is primarily required by + * {@link org.apache.druid.segment.metadata.SegmentMetadataCache} which will run on the Coordinator. + *
    + * Once this class is stable {@link CoordinatorServerView} should be removed. */ @ManageLifecycle public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline @@ -71,20 +77,34 @@ public boolean isAwaitInitializationOnStart() this.baseView = baseView; } + /** + * Internally this class maintains a timeline of {@link ServerSelector}. + * This method returns a newline of the object {@link SegmentLoadInfo}. + * + * @param dataSource dataSoruce + * @return timeline for the given dataSource + */ @Override public VersionedIntervalTimeline getTimeline(DataSource dataSource) { String table = Iterables.getOnlyElement(dataSource.getTableNames()); - synchronized (lock) { - // build a new timeline? - VersionedIntervalTimeline timeline = timelines.get(table); - Collection x = timeline.iterateAllObjects(); - VersionedIntervalTimeline newTimeline = new VersionedIntervalTimeline<>(Comparator.naturalOrder()); - newTimeline.addAll(x.stream().map(v -> new VersionedIntervalTimeline.PartitionChunkEntry( - v.getSegment().getInterval(), v.getSegment().getVersion(), v.getSegment().getShardSpec().createChunk(v.toSegmentLoadInfo()))).iterator()); + VersionedIntervalTimeline timeline; - return newTimeline; + synchronized (lock) { + timeline = timelines.get(table); } + + VersionedIntervalTimeline newTimeline = + new VersionedIntervalTimeline<>(Comparator.naturalOrder()); + newTimeline.addAll( + timeline.iterateAllObjects().stream() + .map(serverSelector -> new VersionedIntervalTimeline.PartitionChunkEntry<>( + serverSelector.getSegment().getInterval(), + serverSelector.getSegment().getVersion(), + serverSelector.getSegment().getShardSpec().createChunk(serverSelector.toSegmentLoadInfo()) + )).iterator()); + + return newTimeline; } @Override diff --git a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java index 6879600878e2..0050edfa8f81 100644 --- a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java @@ -217,6 +217,12 @@ public boolean hasData() return segment.get().hasData(); } + /** + * This conversion is required to make the newer {@link org.apache.druid.client.QueryableCoordinatorServerView} + * implement methods from {@link org.apache.druid.client.CoordinatorTimeline} + * + * @return {@link SegmentLoadInfo} + */ public SegmentLoadInfo toSegmentLoadInfo() { List allServers = getAllServers(); diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index 2e3257e100c1..b8e8c7d5389c 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -26,7 +26,7 @@ import java.util.Objects; /** - * Encapsulates schema information of a dataSource. + * Encapsulates information of a dataSource like schema. */ public class DataSourceInformation { From 2327b12b41382a2b41c20f188cfd55b3509ce9c3 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 00:25:03 +0530 Subject: [PATCH 22/36] Refactor SegmentMetadataCacheTest and BrokerSegmentMetadataCacheTest --- .../SegmentDataCacheConcurrencyTest.java | 6 +- .../metadata/SegmentMetadataCacheCommon.java | 355 ++++++- .../metadata/SegmentMetadataCacheTest.java | 512 +-------- .../BrokerSegmentMetadataCacheTest.java | 990 ++---------------- 4 files changed, 444 insertions(+), 1419 deletions(-) diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index 1746d36b3e16..6a333a9dde02 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -85,9 +85,8 @@ public class SegmentDataCacheConcurrencyTest extends SegmentMetadataCacheCommon { private static final String DATASOURCE = "datasource"; - + static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); private File tmpDir; - private SpecificSegmentsQuerySegmentWalker walker; private TestServerInventoryView inventoryView; private BrokerServerView serverView; private SegmentMetadataCache schema; @@ -96,6 +95,7 @@ public class SegmentDataCacheConcurrencyTest extends SegmentMetadataCacheCommon @Before public void setUp() throws Exception { + setUpCommon(); tmpDir = temporaryFolder.newFolder(); walker = new SpecificSegmentsQuerySegmentWalker(conglomerate); inventoryView = new TestServerInventoryView(); @@ -106,8 +106,10 @@ public void setUp() throws Exception } @After + @Override public void tearDown() throws Exception { + super.tearDown(); exec.shutdownNow(); walker.close(); } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 897f9f0a6463..39bec4cdebf8 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -22,33 +22,59 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.MapInputRowParser; import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; +import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; +import org.apache.druid.query.aggregation.LongSumAggregatorFactory; +import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; +import org.apache.druid.segment.IndexBuilder; +import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.incremental.IncrementalIndexSchema; +import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryStackTests; +import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.log.TestRequestLogger; import org.apache.druid.server.metrics.NoopServiceEmitter; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; +import org.apache.druid.timeline.partition.LinearShardSpec; +import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.junit.Assert; import org.junit.Rule; import org.junit.rules.TemporaryFolder; +import java.io.File; import java.io.IOException; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; public abstract class SegmentMetadataCacheCommon { @@ -69,8 +95,6 @@ public abstract class SegmentMetadataCacheCommon null ); - static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); - final List ROWS1 = ImmutableList.of( createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), createRow(ImmutableMap.of("t", "2000-01-02", "m1", "2.0", "dim1", "10.1")), @@ -83,15 +107,17 @@ public abstract class SegmentMetadataCacheCommon createRow(ImmutableMap.of("t", "2001-01-03", "m1", "6.0")) ); - static QueryRunnerFactoryConglomerate conglomerate; - static Closer resourceCloser; - static QueryToolChestWarehouse queryToolChestWarehouse; + public QueryRunnerFactoryConglomerate conglomerate; + public Closer resourceCloser; + public QueryToolChestWarehouse queryToolChestWarehouse; @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + public SpecificSegmentsQuerySegmentWalker walker; + public TestTimelineServerView serverView; + public List druidServers; - @BeforeClass - public static void setUpClass() + public void setUpCommon() { resourceCloser = Closer.create(); conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); @@ -105,8 +131,174 @@ public > QueryToolChest getToolChest }; } - @AfterClass - public static void tearDownClass() throws IOException + public void setupData() throws Exception + { + final File tmpDir = temporaryFolder.newFolder(); + final QueryableIndex index1 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(ROWS1) + .buildMMappedIndex(); + + final QueryableIndex index2 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "2")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics(new LongSumAggregatorFactory("m1", "m1")) + .withRollup(false) + .build() + ) + .rows(ROWS2) + .buildMMappedIndex(); + + final InputRowSchema rowSchema = new InputRowSchema( + new TimestampSpec("t", null, null), + DimensionsSpec.builder().useSchemaDiscovery(true).build(), + null + ); + final List autoRows1 = ImmutableList.of( + createRow( + ImmutableMap.builder() + .put("t", "2023-01-01T00:00Z") + .put("numbery", 1.1f) + .put("numberyArrays", ImmutableList.of(1L, 2L, 3L)) + .put("stringy", ImmutableList.of("a", "b", "c")) + .put("array", ImmutableList.of(1.1, 2.2, 3.3)) + .put("nested", ImmutableMap.of("x", 1L, "y", 2L)) + .build(), + rowSchema + ) + ); + final List autoRows2 = ImmutableList.of( + createRow( + ImmutableMap.builder() + .put("t", "2023-01-02T00:00Z") + .put("numbery", 1L) + .put("numberyArrays", ImmutableList.of(3.3, 2.2, 3.1)) + .put("stringy", "a") + .put("array", ImmutableList.of(1L, 2L, 3L)) + .put("nested", "hello") + .build(), + rowSchema + ) + ); + + final QueryableIndex indexAuto1 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec(rowSchema.getTimestampSpec()) + .withDimensionsSpec(rowSchema.getDimensionsSpec()) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows1) + .buildMMappedIndex(); + + final QueryableIndex indexAuto2 = IndexBuilder.create() + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec( + new TimestampSpec("t", null, null) + ) + .withDimensionsSpec( + DimensionsSpec.builder().useSchemaDiscovery(true).build() + ) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows2) + .buildMMappedIndex(); + + walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( + DataSegment.builder() + .dataSource(DATASOURCE1) + .interval(Intervals.of("2000/P1Y")) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index1 + ).add( + DataSegment.builder() + .dataSource(DATASOURCE1) + .interval(Intervals.of("2001/P1Y")) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index2 + ).add( + DataSegment.builder() + .dataSource(DATASOURCE2) + .interval(index2.getDataInterval()) + .version("1") + .shardSpec(new LinearShardSpec(0)) + .size(0) + .build(), + index2 + ).add( + DataSegment.builder() + .dataSource(SOME_DATASOURCE) + .interval(Intervals.of("2023-01-01T00Z/P1D")) + .version("1") + .shardSpec(new LinearShardSpec(1)) + .size(0) + .build(), + indexAuto1 + ).add( + DataSegment.builder() + .dataSource(SOME_DATASOURCE) + .interval(Intervals.of("2023-01-02T00Z/P1D")) + .version("1") + .shardSpec(new LinearShardSpec(1)) + .size(0) + .build(), + indexAuto2 + ); + final DataSegment segment1 = new DataSegment( + "foo3", + Intervals.of("2012/2013"), + "version3", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(2, 3), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + final List realtimeSegments = ImmutableList.of(segment1); + serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); + druidServers = serverView.getDruidServers(); + } + + public void tearDown() throws Exception { resourceCloser.close(); } @@ -134,4 +326,145 @@ QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) Suppliers.ofInstance(new DefaultQueryConfig(ImmutableMap.of())) ); } + + public void checkStaleDatasourceRefresh(SegmentMetadataCache schema) throws IOException + { + Set segments = new HashSet<>(); + Set datasources = new HashSet<>(); + datasources.add("wat"); + Assert.assertNull(schema.getDatasource("wat")); + schema.refresh(segments, datasources); + Assert.assertNull(schema.getDatasource("wat")); + } + + public void checkRefreshShouldEmitMetrics( + SegmentMetadataCache schema, + String dataSource, + StubServiceEmitter emitter, + CountDownLatch addSegmentLatch + ) + throws IOException, InterruptedException + { + List segments = ImmutableList.of( + newSegment(dataSource, 1), + newSegment(dataSource, 2) + ); + serverView.addSegment(segments.get(0), ServerType.HISTORICAL); + serverView.addSegment(segments.get(1), ServerType.REALTIME); + Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); + schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(dataSource)); + + emitter.verifyEmitted("metadatacache/refresh/time", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); + emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); + } + + public void checkNullAvailableSegmentMetadata(SegmentMetadataCache schema) throws IOException + { + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // remove one of the segments with datasource "foo" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + } + + public void checkNullDatasource(SegmentMetadataCache schema) throws IOException + { + final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); + final List segments = segmentMetadatas.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // segments contains two segments with datasource "foo" and one with datasource "foo2" + // let's remove the only segment with datasource "foo2" + final DataSegment segmentToRemove = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(segmentToRemove); + schema.removeSegment(segmentToRemove); + + // The following line can cause NPE without segmentMetadata null check in + // SegmentMetadataCache#refreshSegmentsForDataSource + schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); + Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + } + + public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) throws InterruptedException + { + Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); + final List segments = segmentsMetadata.values() + .stream() + .map(AvailableSegmentMetadata::getSegment) + .collect(Collectors.toList()); + Assert.assertEquals(6, segments.size()); + // find the only segment with datasource "foo2" + final DataSegment existingSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + Assert.assertNotNull(existingSegment); + final AvailableSegmentMetadata existingMetadata = segmentsMetadata.get(existingSegment.getId()); + // update AvailableSegmentMetadata of existingSegment with numRows=5 + AvailableSegmentMetadata updatedMetadata = AvailableSegmentMetadata.from(existingMetadata).withNumRows(5).build(); + schema.setAvailableSegmentMetadata(existingSegment.getId(), updatedMetadata); + // find a druidServer holding existingSegment + final Pair pair = druidServers + .stream() + .flatMap(druidServer -> druidServer + .iterateAllSegments() + .stream() + .filter(segment -> segment.getId().equals(existingSegment.getId())) + .map(segment -> Pair.of(druidServer, segment)) + ) + .findAny() + .orElse(null); + Assert.assertNotNull(pair); + final ImmutableDruidServer server = pair.lhs; + Assert.assertNotNull(server); + final DruidServerMetadata druidServerMetadata = server.getMetadata(); + // invoke SegmentMetadataCache#addSegment on existingSegment + schema.addSegment(druidServerMetadata, existingSegment); + segmentsMetadata = schema.getSegmentMetadataSnapshot(); + // get the only segment with datasource "foo2" + final DataSegment currentSegment = segments.stream() + .filter(segment -> segment.getDataSource().equals("foo2")) + .findFirst() + .orElse(null); + final AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); + Assert.assertEquals(updatedMetadata.getSegment().getId(), currentMetadata.getSegment().getId()); + Assert.assertEquals(updatedMetadata.getNumRows(), currentMetadata.getNumRows()); + // numreplicas do not change here since we addSegment with the same server which was serving existingSegment before + Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); + } + + public DataSegment newSegment(String datasource, int partitionId) + { + return new DataSegment( + datasource, + Intervals.of("2012/2013"), + "version1", + null, + ImmutableList.of("dim1", "dim2"), + ImmutableList.of("met1", "met2"), + new NumberedShardSpec(partitionId, 0), + null, + 1, + 100L, + DataSegment.PruneSpecsHolder.DEFAULT + ); + } } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index f300316e6ad8..f016aa0c93a3 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -20,7 +20,6 @@ package org.apache.druid.segment.metadata; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -28,37 +27,22 @@ import com.google.common.collect.Sets; import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.client.InternalQueryConfig; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.InputRowSchema; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; -import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.aggregation.CountAggregatorFactory; -import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; -import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; -import org.apache.druid.segment.IndexBuilder; -import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; -import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryResponse; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -66,20 +50,15 @@ import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.DataSegment.PruneSpecsHolder; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.partition.LinearShardSpec; -import org.apache.druid.timeline.partition.NumberedShardSpec; import org.easymock.EasyMock; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.io.File; import java.io.IOException; import java.util.EnumSet; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -91,188 +70,33 @@ public class SegmentMetadataCacheTest extends SegmentMetadataCacheCommon { // Timeout to allow (rapid) debugging, while not blocking tests with errors. - private static final int WAIT_TIMEOUT_SECS = 6; - - private SpecificSegmentsQuerySegmentWalker walker; - private TestTimelineServerView serverView; - private List druidServers; + private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); + static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); private SegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); - private CountDownLatch refreshLatch = new CountDownLatch(1); - private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); @Before - public void setUp() throws Exception + public void setup() throws Exception { - final File tmpDir = temporaryFolder.newFolder(); - final QueryableIndex index1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(ROWS1) - .buildMMappedIndex(); - - final QueryableIndex index2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "2")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics(new LongSumAggregatorFactory("m1", "m1")) - .withRollup(false) - .build() - ) - .rows(ROWS2) - .buildMMappedIndex(); - - final InputRowSchema rowSchema = new InputRowSchema( - new TimestampSpec("t", null, null), - DimensionsSpec.builder().useSchemaDiscovery(true).build(), - null - ); - final List autoRows1 = ImmutableList.of( - createRow( - ImmutableMap.builder() - .put("t", "2023-01-01T00:00Z") - .put("numbery", 1.1f) - .put("numberyArrays", ImmutableList.of(1L, 2L, 3L)) - .put("stringy", ImmutableList.of("a", "b", "c")) - .put("array", ImmutableList.of(1.1, 2.2, 3.3)) - .put("nested", ImmutableMap.of("x", 1L, "y", 2L)) - .build(), - rowSchema - ) - ); - final List autoRows2 = ImmutableList.of( - createRow( - ImmutableMap.builder() - .put("t", "2023-01-02T00:00Z") - .put("numbery", 1L) - .put("numberyArrays", ImmutableList.of(3.3, 2.2, 3.1)) - .put("stringy", "a") - .put("array", ImmutableList.of(1L, 2L, 3L)) - .put("nested", "hello") - .build(), - rowSchema - ) - ); + setUpCommon(); + setupData(); + } - final QueryableIndex indexAuto1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec(rowSchema.getTimestampSpec()) - .withDimensionsSpec(rowSchema.getDimensionsSpec()) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows1) - .buildMMappedIndex(); - - final QueryableIndex indexAuto2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec( - new TimestampSpec("t", null, null) - ) - .withDimensionsSpec( - DimensionsSpec.builder().useSchemaDiscovery(true).build() - ) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows2) - .buildMMappedIndex(); - - walker = new SpecificSegmentsQuerySegmentWalker(SegmentMetadataCacheCommon.conglomerate).add( - DataSegment.builder() - .dataSource(DATASOURCE1) - .interval(Intervals.of("2000/P1Y")) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index1 - ).add( - DataSegment.builder() - .dataSource(DATASOURCE1) - .interval(Intervals.of("2001/P1Y")) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index2 - ).add( - DataSegment.builder() - .dataSource(DATASOURCE2) - .interval(index2.getDataInterval()) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index2 - ).add( - DataSegment.builder() - .dataSource(SOME_DATASOURCE) - .interval(Intervals.of("2023-01-01T00Z/P1D")) - .version("1") - .shardSpec(new LinearShardSpec(1)) - .size(0) - .build(), - indexAuto1 - ).add( - DataSegment.builder() - .dataSource(SOME_DATASOURCE) - .interval(Intervals.of("2023-01-02T00Z/P1D")) - .version("1") - .shardSpec(new LinearShardSpec(1)) - .size(0) - .build(), - indexAuto2 - ); - final DataSegment segment1 = new DataSegment( - "foo3", - Intervals.of("2012/2013"), - "version3", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(2, 3), - null, - 1, - 100L, - PruneSpecsHolder.DEFAULT - ); - final List realtimeSegments = ImmutableList.of(segment1); - serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); - druidServers = serverView.getDruidServers(); + @After + @Override + public void tearDown() throws Exception + { + super.tearDown(); + if (runningSchema != null) { + runningSchema.stop(); + } + walker.close(); } public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { - return buildSchemaMarkAndTableLatch(SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT); + return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); } public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException @@ -308,48 +132,6 @@ public void markDataSourceAsNeedRebuild(String datasource) return runningSchema; } - public SegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws InterruptedException - { - Preconditions.checkState(runningSchema == null); - runningSchema = new SegmentMetadataCache( - getQueryLifecycleFactory(walker), - serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter() - ) - { - @Override - public void markDataSourceAsNeedRebuild(String datasource) - { - super.markDataSourceAsNeedRebuild(datasource); - markDataSourceLatch.countDown(); - } - - @Override - @VisibleForTesting - public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException - { - super.refresh(segmentsToRefresh, dataSourcesToRebuild); - refreshLatch.countDown(); - } - }; - - runningSchema.start(); - runningSchema.awaitInitialization(); - return runningSchema; - } - - @After - public void tearDown() throws Exception - { - if (runningSchema != null) { - runningSchema.stop(); - } - walker.close(); - } - @Test public void testGetTableMap() throws InterruptedException { @@ -363,8 +145,8 @@ public void testGetTableMap() throws InterruptedException @Test public void testSchemaInit() throws InterruptedException { - SegmentMetadataCache schema2 = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema2.getDatasourceNames()); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); } @Test @@ -511,99 +293,21 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt public void testAvailableSegmentMetadataNumRows() throws InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); - final List segments = segmentsMetadata.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // find the only segment with datasource "foo2" - final DataSegment existingSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - Assert.assertNotNull(existingSegment); - final AvailableSegmentMetadata existingMetadata = segmentsMetadata.get(existingSegment.getId()); - // update AvailableSegmentMetadata of existingSegment with numRows=5 - AvailableSegmentMetadata updatedMetadata = AvailableSegmentMetadata.from(existingMetadata).withNumRows(5).build(); - schema.setAvailableSegmentMetadata(existingSegment.getId(), updatedMetadata); - // find a druidServer holding existingSegment - final Pair pair = druidServers - .stream() - .flatMap(druidServer -> druidServer - .iterateAllSegments() - .stream() - .filter(segment -> segment.getId().equals(existingSegment.getId())) - .map(segment -> Pair.of(druidServer, segment)) - ) - .findAny() - .orElse(null); - Assert.assertNotNull(pair); - final ImmutableDruidServer server = pair.lhs; - Assert.assertNotNull(server); - final DruidServerMetadata druidServerMetadata = server.getMetadata(); - // invoke SegmentMetadataCache#addSegment on existingSegment - schema.addSegment(druidServerMetadata, existingSegment); - segmentsMetadata = schema.getSegmentMetadataSnapshot(); - // get the only segment with datasource "foo2" - final DataSegment currentSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - final AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); - Assert.assertEquals(updatedMetadata.getSegment().getId(), currentMetadata.getSegment().getId()); - Assert.assertEquals(updatedMetadata.getNumRows(), currentMetadata.getNumRows()); - // numreplicas do not change here since we addSegment with the same server which was serving existingSegment before - Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); + checkAvailableSegmentMetadataNumRows(schema); } @Test public void testNullDatasource() throws IOException, InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); - final List segments = segmentMetadatas.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // segments contains two segments with datasource "foo" and one with datasource "foo2" - // let's remove the only segment with datasource "foo2" - final DataSegment segmentToRemove = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - Assert.assertNotNull(segmentToRemove); - schema.removeSegment(segmentToRemove); - - // The following line can cause NPE without segmentMetadata null check in - // SegmentMetadataCache#refreshSegmentsForDataSource - schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); - Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + checkNullDatasource(schema); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); - final List segments = segmentMetadatas.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // remove one of the segments with datasource "foo" - final DataSegment segmentToRemove = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo")) - .findFirst() - .orElse(null); - Assert.assertNotNull(segmentToRemove); - schema.removeSegment(segmentToRemove); - - // The following line can cause NPE without segmentMetadata null check in - // SegmentMetadataCache#refreshSegmentsForDataSource - schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); - Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + checkNullAvailableSegmentMetadata(schema); } @Test @@ -669,7 +373,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -710,7 +414,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -755,7 +459,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -797,7 +501,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -836,7 +540,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -892,7 +596,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -951,7 +655,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -984,7 +688,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -1030,7 +734,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter() @@ -1077,118 +781,6 @@ public void removeServerSegment(final DruidServerMetadata server, final DataSegm Assert.assertEquals(0, metadata.getNumReplicas()); // brokers are not counted as replicas yet } - /** - * Test actions on the cache. The current design of the cache makes testing far harder - * than it should be. - * - * - The cache is refreshed on a schedule. - * - Datasources are added to the refresh queue via an unsynchronized thread. - * - The refresh loop always refreshes since one of the segments is dynamic. - * - * The use of latches tries to keep things synchronized, but there are many - * moving parts. A simpler technique is sorely needed. - */ - @Test - public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws InterruptedException - { - SegmentMetadataCache schema3 = buildSchemaMarkAndRefreshLatch(); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DataSourceInformation fooTable = schema3.getDatasource("foo"); - Assert.assertNotNull(fooTable); - - markDataSourceLatch = new CountDownLatch(1); - refreshLatch = new CountDownLatch(1); - final DataSegment someNewBrokerSegment = new DataSegment( - "foo", - Intervals.of("2012/2013"), - "version1", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(2, 3), - null, - 1, - 100L, - PruneSpecsHolder.DEFAULT - ); - serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); - Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for build twice - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) - refreshLatch = new CountDownLatch(1); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - - fooTable = schema3.getDatasource("foo"); - Assert.assertNotNull(fooTable); - - // now remove it - markDataSourceLatch = new CountDownLatch(1); - refreshLatch = new CountDownLatch(1); - serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); - - Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for build twice - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) - refreshLatch = new CountDownLatch(1); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - - fooTable = schema3.getDatasource("foo"); - Assert.assertNotNull(fooTable); - } - - @Test - public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throws InterruptedException - { - SegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DataSourceInformation fooTable = schema.getDatasource("foo"); - Assert.assertNotNull(fooTable); - - markDataSourceLatch = new CountDownLatch(1); - refreshLatch = new CountDownLatch(1); - final DataSegment someNewBrokerSegment = new DataSegment( - "foo", - Intervals.of("2012/2013"), - "version1", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(2, 3), - null, - 1, - 100L, - PruneSpecsHolder.DEFAULT - ); - serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); - - Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for build twice - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) - refreshLatch = new CountDownLatch(1); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - - fooTable = schema.getDatasource("foo"); - Assert.assertNotNull(fooTable); - - // now remove it - markDataSourceLatch = new CountDownLatch(1); - refreshLatch = new CountDownLatch(1); - serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); - - Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for build twice - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated) - refreshLatch = new CountDownLatch(1); - Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - - fooTable = schema.getDatasource("foo"); - Assert.assertNotNull(fooTable); - } - /** * Ensure that the BrokerInternalQueryConfig context is honored for this internally generated SegmentMetadata Query */ @@ -1235,7 +827,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception SegmentMetadataCache mySchema = new SegmentMetadataCache( factoryMock, serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), internalQueryConfig, new NoopServiceEmitter() @@ -1351,24 +943,19 @@ public void testSegmentMetadataFallbackType() public void testStaleDatasourceRefresh() throws IOException, InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Set segments = new HashSet<>(); - Set datasources = new HashSet<>(); - datasources.add("wat"); - Assert.assertNull(schema.getDatasource("wat")); - schema.refresh(segments, datasources); - Assert.assertNull(schema.getDatasource("wat")); + checkStaleDatasourceRefresh(schema); } @Test public void testRefreshShouldEmitMetrics() throws InterruptedException, IOException { - String datasource = "xyz"; + String dataSource = "xyz"; CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, - SegmentMetadataCacheCommon.SEGMENT_CACHE_CONFIG_DEFAULT, + SEGMENT_CACHE_CONFIG_DEFAULT, new NoopEscalator(), new InternalQueryConfig(), emitter @@ -1378,7 +965,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { + if (dataSource.equals(segment.getDataSource())) { addSegmentLatch.countDown(); } } @@ -1390,33 +977,6 @@ public void removeSegment(final DataSegment segment) } }; - List segments = ImmutableList.of( - newSegment(datasource, 1), - newSegment(datasource, 2) - ); - serverView.addSegment(segments.get(0), ServerType.HISTORICAL); - serverView.addSegment(segments.get(1), ServerType.REALTIME); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(datasource)); - - emitter.verifyEmitted("metadatacache/refresh/time", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); - emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); - } - - private static DataSegment newSegment(String datasource, int partitionId) - { - return new DataSegment( - datasource, - Intervals.of("2012/2013"), - "version1", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(partitionId, 0), - null, - 1, - 100L, - PruneSpecsHolder.DEFAULT - ); + checkRefreshShouldEmitMetrics(schema, dataSource, emitter, addSegmentLatch); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index eeec64ca296f..8aa6a36f8f84 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -30,39 +30,19 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.coordinator.NoopCoordinatorClient; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.InputRowSchema; -import org.apache.druid.data.input.impl.DimensionsSpec; -import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.guava.Sequences; -import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.query.DataSource; -import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.aggregation.CountAggregatorFactory; -import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; -import org.apache.druid.query.aggregation.LongSumAggregatorFactory; -import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; -import org.apache.druid.query.metadata.metadata.ColumnAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; -import org.apache.druid.segment.IndexBuilder; -import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.TestHelper; -import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.RowSignature; -import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.join.JoinConditionAnalysis; import org.apache.druid.segment.join.Joinable; import org.apache.druid.segment.join.JoinableFactory; @@ -70,14 +50,11 @@ import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; -import org.apache.druid.segment.metadata.TestTimelineServerView; -import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.segment.metadata.SegmentMetadataCacheCommon; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryResponse; -import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SegmentManager; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; @@ -86,26 +63,18 @@ import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.sql.calcite.table.DatasourceTable; import org.apache.druid.sql.calcite.table.DruidTable; -import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.TestDataBuilder; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import org.apache.druid.timeline.partition.LinearShardSpec; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.easymock.EasyMock; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import java.io.File; import java.io.IOException; import java.util.EnumSet; -import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -116,45 +85,29 @@ // test polling from coordinator and when coordinator doesn't return anything // test the result -public class BrokerSegmentMetadataCacheTest extends CalciteTestBase +public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon { private final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); // Timeout to allow (rapid) debugging, while not blocking tests with errors. private static final int WAIT_TIMEOUT_SECS = 6; - private SpecificSegmentsQuerySegmentWalker walker; - private TestTimelineServerView serverView; - private List druidServers; private BrokerSegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); private CountDownLatch refreshLatch = new CountDownLatch(1); - @Rule - public TemporaryFolder temporaryFolder = new TemporaryFolder(); SegmentManager segmentManager; Set segmentDataSourceNames; Set joinableDataSourceNames; JoinableFactory globalTableJoinable; CountDownLatch getDatasourcesLatch = new CountDownLatch(1); - private QueryRunnerFactoryConglomerate conglomerate; - private Closer resourceCloser; private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); - static final List ROWS1 = ImmutableList.of( - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-01", "m1", "1.0", "dim1", "")), - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-02", "m1", "2.0", "dim1", "10.1")), - TestDataBuilder.createRow(ImmutableMap.of("t", "2000-01-03", "m1", "3.0", "dim1", "2")) - ); - - static final List ROWS2 = ImmutableList.of( - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-01", "m1", "4.0", "dim2", ImmutableList.of("a"))), - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-02", "m1", "5.0", "dim2", ImmutableList.of("abc"))), - TestDataBuilder.createRow(ImmutableMap.of("t", "2001-01-03", "m1", "6.0")) - ); @Before public void setUp() throws Exception { + setUpCommon(); + setupData(); segmentDataSourceNames = Sets.newConcurrentHashSet(); joinableDataSourceNames = Sets.newConcurrentHashSet(); @@ -186,173 +139,17 @@ public Optional build( return Optional.empty(); } }; + } - resourceCloser = Closer.create(); - conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); - - final File tmpDir = temporaryFolder.newFolder(); - final QueryableIndex index1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(ROWS1) - .buildMMappedIndex(); - - final QueryableIndex index2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "2")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics(new LongSumAggregatorFactory("m1", "m1")) - .withRollup(false) - .build() - ) - .rows(ROWS2) - .buildMMappedIndex(); - - final InputRowSchema rowSchema = new InputRowSchema( - new TimestampSpec("t", null, null), - DimensionsSpec.builder().useSchemaDiscovery(true).build(), - null - ); - final List autoRows1 = ImmutableList.of( - TestDataBuilder.createRow( - ImmutableMap.builder() - .put("t", "2023-01-01T00:00Z") - .put("numbery", 1.1f) - .put("numberyArrays", ImmutableList.of(1L, 2L, 3L)) - .put("stringy", ImmutableList.of("a", "b", "c")) - .put("array", ImmutableList.of(1.1, 2.2, 3.3)) - .put("nested", ImmutableMap.of("x", 1L, "y", 2L)) - .build(), - rowSchema - ) - ); - final List autoRows2 = ImmutableList.of( - TestDataBuilder.createRow( - ImmutableMap.builder() - .put("t", "2023-01-02T00:00Z") - .put("numbery", 1L) - .put("numberyArrays", ImmutableList.of(3.3, 2.2, 3.1)) - .put("stringy", "a") - .put("array", ImmutableList.of(1L, 2L, 3L)) - .put("nested", "hello") - .build(), - rowSchema - ) - ); - - final QueryableIndex indexAuto1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec(rowSchema.getTimestampSpec()) - .withDimensionsSpec(rowSchema.getDimensionsSpec()) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows1) - .buildMMappedIndex(); - - final QueryableIndex indexAuto2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec( - new TimestampSpec("t", null, null) - ) - .withDimensionsSpec( - DimensionsSpec.builder().useSchemaDiscovery(true).build() - ) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows2) - .buildMMappedIndex(); - - walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( - DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE1) - .interval(Intervals.of("2000/P1Y")) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index1 - ).add( - DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE1) - .interval(Intervals.of("2001/P1Y")) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index2 - ).add( - DataSegment.builder() - .dataSource(CalciteTests.DATASOURCE2) - .interval(index2.getDataInterval()) - .version("1") - .shardSpec(new LinearShardSpec(0)) - .size(0) - .build(), - index2 - ).add( - DataSegment.builder() - .dataSource(CalciteTests.SOME_DATASOURCE) - .interval(Intervals.of("2023-01-01T00Z/P1D")) - .version("1") - .shardSpec(new LinearShardSpec(1)) - .size(0) - .build(), - indexAuto1 - ).add( - DataSegment.builder() - .dataSource(CalciteTests.SOME_DATASOURCE) - .interval(Intervals.of("2023-01-02T00Z/P1D")) - .version("1") - .shardSpec(new LinearShardSpec(1)) - .size(0) - .build(), - indexAuto2 - ); - final DataSegment segment1 = new DataSegment( - "foo3", - Intervals.of("2012/2013"), - "version3", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(2, 3), - null, - 1, - 100L, - DataSegment.PruneSpecsHolder.DEFAULT - ); - final List realtimeSegments = ImmutableList.of(segment1); - serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); - druidServers = serverView.getDruidServers(); + @After + @Override + public void tearDown() throws Exception + { + super.tearDown(); + if (runningSchema != null) { + runningSchema.stop(); + } + walker.close(); } public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException @@ -431,16 +228,6 @@ public void refresh(final Set segmentsToRefresh, final Set da return runningSchema; } - @After - public void tearDown() throws Exception - { - if (runningSchema != null) { - runningSchema.stop(); - } - walker.close(); - resourceCloser.close(); - } - @Test public void testGetTableMap() throws InterruptedException { @@ -454,8 +241,8 @@ public void testGetTableMap() throws InterruptedException @Test public void testSchemaInit() throws InterruptedException { - BrokerSegmentMetadataCache schema2 = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema2.getDatasourceNames()); + BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema.getDatasourceNames()); } @Test @@ -614,586 +401,21 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt public void testAvailableSegmentMetadataNumRows() throws InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); - final List segments = segmentsMetadata.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // find the only segment with datasource "foo2" - final DataSegment existingSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - Assert.assertNotNull(existingSegment); - final AvailableSegmentMetadata existingMetadata = segmentsMetadata.get(existingSegment.getId()); - // update AvailableSegmentMetadata of existingSegment with numRows=5 - AvailableSegmentMetadata updatedMetadata = AvailableSegmentMetadata.from(existingMetadata).withNumRows(5).build(); - schema.setAvailableSegmentMetadata(existingSegment.getId(), updatedMetadata); - // find a druidServer holding existingSegment - final Pair pair = druidServers - .stream() - .flatMap(druidServer -> druidServer - .iterateAllSegments() - .stream() - .filter(segment -> segment.getId().equals(existingSegment.getId())) - .map(segment -> Pair.of(druidServer, segment)) - ) - .findAny() - .orElse(null); - Assert.assertNotNull(pair); - final ImmutableDruidServer server = pair.lhs; - Assert.assertNotNull(server); - final DruidServerMetadata druidServerMetadata = server.getMetadata(); - // invoke SegmentMetadataCache#addSegment on existingSegment - schema.addSegment(druidServerMetadata, existingSegment); - segmentsMetadata = schema.getSegmentMetadataSnapshot(); - // get the only segment with datasource "foo2" - final DataSegment currentSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - final AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); - Assert.assertEquals(updatedMetadata.getSegment().getId(), currentMetadata.getSegment().getId()); - Assert.assertEquals(updatedMetadata.getNumRows(), currentMetadata.getNumRows()); - // numreplicas do not change here since we addSegment with the same server which was serving existingSegment before - Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); + checkAvailableSegmentMetadataNumRows(schema); } @Test public void testNullDatasource() throws IOException, InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); - final List segments = segmentMetadatas.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // segments contains two segments with datasource "foo" and one with datasource "foo2" - // let's remove the only segment with datasource "foo2" - final DataSegment segmentToRemove = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo2")) - .findFirst() - .orElse(null); - Assert.assertNotNull(segmentToRemove); - schema.removeSegment(segmentToRemove); - - // The following line can cause NPE without segmentMetadata null check in - // SegmentMetadataCache#refreshSegmentsForDataSource - schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); - Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); + checkNullDatasource(schema); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); - final List segments = segmentMetadatas.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - Assert.assertEquals(6, segments.size()); - // remove one of the segments with datasource "foo" - final DataSegment segmentToRemove = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo")) - .findFirst() - .orElse(null); - Assert.assertNotNull(segmentToRemove); - schema.removeSegment(segmentToRemove); - - // The following line can cause NPE without segmentMetadata null check in - // SegmentMetadataCache#refreshSegmentsForDataSource - schema.refreshSegments(segments.stream().map(DataSegment::getId).collect(Collectors.toSet())); - Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); - } - - @Test - public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException - { - BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); - final List segments = segmentsMetadata.values() - .stream() - .map(AvailableSegmentMetadata::getSegment) - .collect(Collectors.toList()); - // find the only realtime segment with datasource "foo3" - final DataSegment existingSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo3")) - .findFirst() - .orElse(null); - Assert.assertNotNull(existingSegment); - final AvailableSegmentMetadata metadata = segmentsMetadata.get(existingSegment.getId()); - Assert.assertEquals(1L, metadata.isRealtime()); - // get the historical server - final ImmutableDruidServer historicalServer = druidServers.stream() - .filter(s -> s.getType().equals(ServerType.HISTORICAL)) - .findAny() - .orElse(null); - - Assert.assertNotNull(historicalServer); - final DruidServerMetadata historicalServerMetadata = historicalServer.getMetadata(); - - // add existingSegment to historical - schema.addSegment(historicalServerMetadata, existingSegment); - segmentsMetadata = schema.getSegmentMetadataSnapshot(); - // get the segment with datasource "foo3" - DataSegment currentSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo3")) - .findFirst() - .orElse(null); - Assert.assertNotNull(currentSegment); - AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId()); - Assert.assertEquals(0L, currentMetadata.isRealtime()); - - ImmutableDruidServer realtimeServer = druidServers.stream() - .filter(s -> s.getType().equals(ServerType.REALTIME)) - .findAny() - .orElse(null); - Assert.assertNotNull(realtimeServer); - // drop existingSegment from realtime task - schema.removeServerSegment(realtimeServer.getMetadata(), existingSegment); - segmentsMetadata = schema.getSegmentMetadataSnapshot(); - currentSegment = segments.stream() - .filter(segment -> segment.getDataSource().equals("foo3")) - .findFirst() - .orElse(null); - Assert.assertNotNull(currentSegment); - currentMetadata = segmentsMetadata.get(currentSegment.getId()); - Assert.assertEquals(0L, currentMetadata.isRealtime()); - } - - @Test - public void testSegmentAddedCallbackAddNewHistoricalSegment() throws InterruptedException - { - String datasource = "newSegmentAddTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - }; - - serverView.addSegment(newSegment(datasource, 1), ServerType.HISTORICAL); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(1, metadatas.size()); - AvailableSegmentMetadata metadata = metadatas.get(0); - Assert.assertEquals(0, metadata.isRealtime()); - Assert.assertEquals(0, metadata.getNumRows()); - Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); - } - - @Test - public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedException - { - String datasource = "newSegmentAddTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(2); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - }; - - DataSegment segment = newSegment(datasource, 1); - serverView.addSegment(segment, ServerType.REALTIME); - serverView.addSegment(segment, ServerType.HISTORICAL); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(1, metadatas.size()); - AvailableSegmentMetadata metadata = metadatas.get(0); - Assert.assertEquals(0, metadata.isRealtime()); // realtime flag is unset when there is any historical - Assert.assertEquals(0, metadata.getNumRows()); - Assert.assertEquals(2, metadata.getNumReplicas()); - Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); - Assert.assertFalse(schema.getMutableSegments().contains(metadata.getSegment().getId())); - } - - @Test - public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedException - { - String datasource = "newSegmentAddTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - }; - - serverView.addSegment(newSegment(datasource, 1), ServerType.REALTIME); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(1, metadatas.size()); - AvailableSegmentMetadata metadata = metadatas.get(0); - Assert.assertEquals(1, metadata.isRealtime()); - Assert.assertEquals(0, metadata.getNumRows()); - Assert.assertTrue(schema.getSegmentsNeedingRefresh().contains(metadata.getSegment().getId())); - Assert.assertTrue(schema.getMutableSegments().contains(metadata.getSegment().getId())); - } - - @Test - public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedException - { - String datasource = "newSegmentAddTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - }; - - serverView.addSegment(newSegment(datasource, 1), ServerType.BROKER); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(6, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(0, metadatas.size()); - Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); - } - - @Test - public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws InterruptedException, IOException - { - String datasource = "segmentRemoveTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - CountDownLatch removeSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - - @Override - public void removeSegment(final DataSegment segment) - { - super.removeSegment(segment); - if (datasource.equals(segment.getDataSource())) { - removeSegmentLatch.countDown(); - } - } - }; - - DataSegment segment = newSegment(datasource, 1); - serverView.addSegment(segment, ServerType.REALTIME); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - schema.refresh(Sets.newHashSet(segment.getId()), Sets.newHashSet(datasource)); - - serverView.removeSegment(segment, ServerType.REALTIME); - Assert.assertTrue(removeSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(6, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(0, metadatas.size()); - Assert.assertFalse(schema.getSegmentsNeedingRefresh().contains(segment.getId())); - Assert.assertFalse(schema.getMutableSegments().contains(segment.getId())); - Assert.assertFalse(schema.getDataSourcesNeedingRebuild().contains(datasource)); - Assert.assertFalse(schema.getDatasourceNames().contains(datasource)); - } - - @Test - public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws InterruptedException, IOException - { - String datasource = "segmentRemoveTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(2); - CountDownLatch removeSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - - @Override - public void removeSegment(final DataSegment segment) - { - super.removeSegment(segment); - if (datasource.equals(segment.getDataSource())) { - removeSegmentLatch.countDown(); - } - } - }; - - List segments = ImmutableList.of( - newSegment(datasource, 1), - newSegment(datasource, 2) - ); - serverView.addSegment(segments.get(0), ServerType.REALTIME); - serverView.addSegment(segments.get(1), ServerType.HISTORICAL); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(datasource)); - - serverView.removeSegment(segments.get(0), ServerType.REALTIME); - Assert.assertTrue(removeSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(1, metadatas.size()); - Assert.assertFalse(schema.getSegmentsNeedingRefresh().contains(segments.get(0).getId())); - Assert.assertFalse(schema.getMutableSegments().contains(segments.get(0).getId())); - Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); - Assert.assertTrue(schema.getDatasourceNames().contains(datasource)); - } - - @Test - public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws InterruptedException - { - String datasource = "serverSegmentRemoveTest"; - CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.removeServerSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - removeServerSegmentLatch.countDown(); - } - } - }; - - serverView.addSegment(newSegment(datasource, 1), ServerType.BROKER); - - serverView.removeSegment(newSegment(datasource, 1), ServerType.HISTORICAL); - Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(6, schema.getTotalSegments()); - } - - @Test - public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws InterruptedException - { - String datasource = "serverSegmentRemoveTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - - @Override - public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.removeServerSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - removeServerSegmentLatch.countDown(); - } - } - }; - - DataSegment segment = newSegment(datasource, 1); - serverView.addSegment(segment, ServerType.HISTORICAL); - serverView.addSegment(segment, ServerType.BROKER); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - serverView.removeSegment(segment, ServerType.BROKER); - Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - Assert.assertTrue(schema.getDataSourcesNeedingRebuild().contains(datasource)); - } - - @Test - public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws InterruptedException - { - String datasource = "serverSegmentRemoveTest"; - CountDownLatch addSegmentLatch = new CountDownLatch(1); - CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( - CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), - serverView, - SEGMENT_CACHE_CONFIG_DEFAULT, - new NoopEscalator(), - new InternalQueryConfig(), - new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() - ) - { - @Override - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - addSegmentLatch.countDown(); - } - } - - @Override - public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) - { - super.removeServerSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { - removeServerSegmentLatch.countDown(); - } - } - }; - - DataSegment segment = newSegment(datasource, 1); - serverView.addSegment(segment, ServerType.HISTORICAL); - serverView.addSegment(segment, ServerType.BROKER); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - - serverView.removeSegment(segment, ServerType.HISTORICAL); - Assert.assertTrue(removeServerSegmentLatch.await(1, TimeUnit.SECONDS)); - - Assert.assertEquals(7, schema.getTotalSegments()); - List metadatas = schema - .getSegmentMetadataSnapshot() - .values() - .stream() - .filter(metadata -> datasource.equals(metadata.getSegment().getDataSource())) - .collect(Collectors.toList()); - Assert.assertEquals(1, metadatas.size()); - AvailableSegmentMetadata metadata = metadatas.get(0); - Assert.assertEquals(0, metadata.isRealtime()); - Assert.assertEquals(0, metadata.getNumRows()); - Assert.assertEquals(0, metadata.getNumReplicas()); // brokers are not counted as replicas yet + checkNullAvailableSegmentMetadata(schema); } /** @@ -1210,10 +432,14 @@ public void removeServerSegment(final DruidServerMetadata server, final DataSegm @Test public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws InterruptedException { - BrokerSegmentMetadataCache schema3 = buildSchemaMarkAndRefreshLatch(); + BrokerSegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DatasourceTable.PhysicalDatasourceMetadata fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertFalse(fooTable.isJoinable()); + Assert.assertFalse(fooTable.isBroadcast()); markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); @@ -1230,6 +456,8 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte 100L, DataSegment.PruneSpecsHolder.DEFAULT ); + segmentDataSourceNames.add("foo"); + joinableDataSourceNames.add("foo"); serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); // wait for build twice @@ -1238,12 +466,18 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + Assert.assertTrue(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertTrue(fooTable.isJoinable()); + Assert.assertTrue(fooTable.isBroadcast()); // now remove it markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); + joinableDataSourceNames.remove("foo"); + segmentDataSourceNames.remove("foo"); serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1253,8 +487,12 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema3.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertFalse(fooTable.isJoinable()); + Assert.assertFalse(fooTable.isBroadcast()); } @Test @@ -1264,6 +502,11 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertFalse(fooTable.isJoinable()); + Assert.assertFalse(fooTable.isBroadcast()); markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); @@ -1280,6 +523,7 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw 100L, DataSegment.PruneSpecsHolder.DEFAULT ); + segmentDataSourceNames.add("foo"); serverView.addSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1291,10 +535,17 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + // Should not be a GlobalTableDataSource for now, because isGlobal is couple with joinability. Ideally this will be + // changed in the future and we should expect. + Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertTrue(fooTable.isBroadcast()); + Assert.assertFalse(fooTable.isJoinable()); // now remove it markDataSourceLatch = new CountDownLatch(1); refreshLatch = new CountDownLatch(1); + segmentDataSourceNames.remove("foo"); serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER); Assert.assertTrue(markDataSourceLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); @@ -1306,6 +557,10 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw fooTable = schema.getPhysicalDatasourceMetadata("foo"); Assert.assertNotNull(fooTable); + Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); + Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); + Assert.assertFalse(fooTable.isBroadcast()); + Assert.assertFalse(fooTable.isJoinable()); } /** @@ -1375,115 +630,17 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception } - @Test - public void testSegmentMetadataColumnType() - { - // Verify order is preserved. - final LinkedHashMap columns = new LinkedHashMap<>(); - columns.put( - "a", - new ColumnAnalysis(ColumnType.STRING, ColumnType.STRING.asTypeString(), false, true, 1234, 26, "a", "z", null) - ); - - columns.put( - "count", - new ColumnAnalysis(ColumnType.LONG, ColumnType.LONG.asTypeString(), false, true, 1234, 26, "a", "z", null) - ); - - columns.put( - "b", - new ColumnAnalysis(ColumnType.DOUBLE, ColumnType.DOUBLE.asTypeString(), false, true, 1234, 26, null, null, null) - ); - - RowSignature signature = SegmentMetadataCache.analysisToRowSignature( - new SegmentAnalysis( - "id", - ImmutableList.of(Intervals.utc(1L, 2L)), - columns, - 1234, - 100, - null, - null, - null, - null - ) - ); - - Assert.assertEquals( - RowSignature.builder() - .add("a", ColumnType.STRING) - .add("count", ColumnType.LONG) - .add("b", ColumnType.DOUBLE) - .build(), - signature - ); - } - - @Test - public void testSegmentMetadataFallbackType() - { - RowSignature signature = SegmentMetadataCache.analysisToRowSignature( - new SegmentAnalysis( - "id", - ImmutableList.of(Intervals.utc(1L, 2L)), - new LinkedHashMap<>( - ImmutableMap.of( - "a", - new ColumnAnalysis( - null, - ColumnType.STRING.asTypeString(), - false, - true, - 1234, - 26, - "a", - "z", - null - ), - "count", - new ColumnAnalysis( - null, - ColumnType.LONG.asTypeString(), - false, - true, - 1234, - 26, - "a", - "z", - null - ) - ) - ), - 1234, - 100, - null, - null, - null, - null - ) - ); - Assert.assertEquals( - RowSignature.builder().add("a", ColumnType.STRING).add("count", ColumnType.LONG).build(), - signature - ); - } - @Test public void testStaleDatasourceRefresh() throws IOException, InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Set segments = new HashSet<>(); - Set datasources = new HashSet<>(); - datasources.add("wat"); - Assert.assertNull(schema.getDatasource("wat")); - schema.refresh(segments, datasources); - Assert.assertNull(schema.getDatasource("wat")); + checkStaleDatasourceRefresh(schema); } @Test public void testRefreshShouldEmitMetrics() throws InterruptedException, IOException { - String datasource = "xyz"; + String dataSource = "xyz"; CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( @@ -1501,7 +658,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept public void addSegment(final DruidServerMetadata server, final DataSegment segment) { super.addSegment(server, segment); - if (datasource.equals(segment.getDataSource())) { + if (dataSource.equals(segment.getDataSource())) { addSegmentLatch.countDown(); } } @@ -1513,33 +670,6 @@ public void removeSegment(final DataSegment segment) } }; - List segments = ImmutableList.of( - newSegment(datasource, 1), - newSegment(datasource, 2) - ); - serverView.addSegment(segments.get(0), ServerType.HISTORICAL); - serverView.addSegment(segments.get(1), ServerType.REALTIME); - Assert.assertTrue(addSegmentLatch.await(1, TimeUnit.SECONDS)); - schema.refresh(segments.stream().map(DataSegment::getId).collect(Collectors.toSet()), Sets.newHashSet(datasource)); - - emitter.verifyEmitted("metadatacache/refresh/time", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); - emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, datasource), 1); - } - - private static DataSegment newSegment(String datasource, int partitionId) - { - return new DataSegment( - datasource, - Intervals.of("2012/2013"), - "version1", - null, - ImmutableList.of("dim1", "dim2"), - ImmutableList.of("met1", "met2"), - new NumberedShardSpec(partitionId, 0), - null, - 1, - 100L, - DataSegment.PruneSpecsHolder.DEFAULT - ); + checkRefreshShouldEmitMetrics(schema, dataSource, emitter, addSegmentLatch); } } From d4ece6a138a297c4c21a7c0905b9dbd4ca367193 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 14:15:58 +0530 Subject: [PATCH 23/36] Test fetching ds schema from coordinator in BrokerSegmentMetadataCacheTest --- .../metadata/SegmentMetadataCache.java | 2 + .../metadata/SegmentMetadataCacheCommon.java | 74 +++++++- .../metadata/SegmentMetadataCacheTest.java | 42 +---- .../schema/BrokerSegmentMetadataCache.java | 7 +- .../BrokerSegmentMetadataCacheTest.java | 162 ++++++++++++------ 5 files changed, 187 insertions(+), 100 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index e5709fac0dbd..6d80bd9390a8 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -61,7 +61,9 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.Types; +import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.QueryResponse; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.security.Access; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 39bec4cdebf8..096e96ed39e7 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -31,43 +31,60 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.metrics.StubServiceEmitter; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; import org.apache.druid.query.DruidMetrics; import org.apache.druid.query.Query; +import org.apache.druid.query.QueryContexts; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.TableDataSource; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.aggregation.hyperloglog.HyperUniquesAggregatorFactory; +import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; +import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.QueryableIndexStorageAdapter; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.incremental.IncrementalIndexSchema; +import org.apache.druid.segment.join.table.IndexedTable; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.QueryResponse; import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.log.TestRequestLogger; import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.server.security.Access; +import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.AuthConfig; import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.partition.LinearShardSpec; import org.apache.druid.timeline.partition.NumberedShardSpec; +import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Rule; import org.junit.rules.TemporaryFolder; import java.io.File; import java.io.IOException; +import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -117,6 +134,13 @@ public abstract class SegmentMetadataCacheCommon public TestTimelineServerView serverView; public List druidServers; + public QueryableIndex index1; + public QueryableIndex index2; + + public QueryableIndex indexAuto1; + public QueryableIndex indexAuto2; + public DataSegment realtimeSegment1; + public void setUpCommon() { resourceCloser = Closer.create(); @@ -134,7 +158,7 @@ public > QueryToolChest getToolChest public void setupData() throws Exception { final File tmpDir = temporaryFolder.newFolder(); - final QueryableIndex index1 = IndexBuilder.create() + index1 = IndexBuilder.create() .tmpDir(new File(tmpDir, "1")) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( @@ -150,7 +174,7 @@ public void setupData() throws Exception .rows(ROWS1) .buildMMappedIndex(); - final QueryableIndex index2 = IndexBuilder.create() + index2 = IndexBuilder.create() .tmpDir(new File(tmpDir, "2")) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( @@ -194,7 +218,7 @@ public void setupData() throws Exception ) ); - final QueryableIndex indexAuto1 = IndexBuilder.create() + indexAuto1 = IndexBuilder.create() .tmpDir(new File(tmpDir, "1")) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( @@ -212,7 +236,7 @@ public void setupData() throws Exception .rows(autoRows1) .buildMMappedIndex(); - final QueryableIndex indexAuto2 = IndexBuilder.create() + indexAuto2 = IndexBuilder.create() .tmpDir(new File(tmpDir, "1")) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .schema( @@ -280,7 +304,7 @@ public void setupData() throws Exception .build(), indexAuto2 ); - final DataSegment segment1 = new DataSegment( + realtimeSegment1 = new DataSegment( "foo3", Intervals.of("2012/2013"), "version3", @@ -293,7 +317,8 @@ public void setupData() throws Exception 100L, DataSegment.PruneSpecsHolder.DEFAULT ); - final List realtimeSegments = ImmutableList.of(segment1); + + final List realtimeSegments = ImmutableList.of(realtimeSegment1); serverView = new TestTimelineServerView(walker.getSegments(), realtimeSegments); druidServers = serverView.getDruidServers(); } @@ -451,6 +476,43 @@ public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) th Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); } + public void checkRunSegmentMetadataQueryWithContext(SegmentMetadataCache schema, QueryLifecycleFactory factoryMock, QueryLifecycle lifecycleMock) + { + Map queryContext = ImmutableMap.of( + QueryContexts.PRIORITY_KEY, 5, + QueryContexts.BROKER_PARALLEL_MERGE_KEY, false + ); + + DataSegment segment = newSegment("test", 0); + List segmentIterable = ImmutableList.of(segment.getId()); + + // This is the query that we expect this method to create. We will be testing that it matches the query generated by the method under test. + SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( + new TableDataSource(segment.getDataSource()), + new MultipleSpecificSegmentSpec( + segmentIterable.stream() + .map(SegmentId::toDescriptor).collect(Collectors.toList())), + new AllColumnIncluderator(), + false, + queryContext, + EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), + false, + null, + null + ); + + EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); + // This is the mat of the test, making sure that the query created by the method under test matches the expected query, specifically the operator configured context + EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) + .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); + + EasyMock.replay(factoryMock, lifecycleMock); + + schema.runSegmentMetadataQuery(segmentIterable); + + EasyMock.verify(factoryMock, lifecycleMock); + } + public DataSegment newSegment(String datasource, int partitionId) { return new DataSegment( diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index f016aa0c93a3..4c969464a6ab 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -142,13 +142,6 @@ public void testGetTableMap() throws InterruptedException Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), tableNames); } - @Test - public void testSchemaInit() throws InterruptedException - { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); - } - @Test public void testGetTableMapFoo() throws InterruptedException { @@ -787,11 +780,6 @@ public void removeServerSegment(final DruidServerMetadata server, final DataSegm @Test public void testRunSegmentMetadataQueryWithContext() throws Exception { - Map queryContext = ImmutableMap.of( - QueryContexts.PRIORITY_KEY, 5, - QueryContexts.BROKER_PARALLEL_MERGE_KEY, false - ); - String brokerInternalQueryConfigJson = "{\"context\": { \"priority\": 5} }"; TestHelper.makeJsonMapper(); @@ -802,24 +790,6 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception InternalQueryConfig.class ); - DataSegment segment = newSegment("test", 0); - List segmentIterable = ImmutableList.of(segment.getId()); - - // This is the query that we expect this method to create. We will be testing that it matches the query generated by the method under test. - SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( - new TableDataSource(segment.getDataSource()), - new MultipleSpecificSegmentSpec( - segmentIterable.stream() - .map(SegmentId::toDescriptor).collect(Collectors.toList())), - new AllColumnIncluderator(), - false, - queryContext, - EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), - false, - null, - null - ); - QueryLifecycleFactory factoryMock = EasyMock.createMock(QueryLifecycleFactory.class); QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); @@ -833,17 +803,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception new NoopServiceEmitter() ); - EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); - // This is the mat of the test, making sure that the query created by the method under test matches the expected query, specifically the operator configured context - EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) - .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); - - EasyMock.replay(factoryMock, lifecycleMock); - - mySchema.runSegmentMetadataQuery(segmentIterable); - - EasyMock.verify(factoryMock, lifecycleMock); - + checkRunSegmentMetadataQueryWithContext(mySchema, factoryMock, lifecycleMock); } @Test diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 8c2369aaab62..921bbcf85461 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -104,11 +105,7 @@ public void refresh(final Set segmentsToRefresh, final Set da tables.putAll(polledDataSourceMetadata); // Remove segments of the dataSource from refresh list for which we received schema from the Coordinator. - segmentsToRefresh.forEach(segment -> { - if (polledDataSourceMetadata.containsKey(segment.getDataSource())) { - segmentsToRefresh.remove(segment); - } - }); + segmentsToRefresh.removeIf(segmentId -> polledDataSourceMetadata.containsKey(segmentId.getDataSource())); // Refresh the remaining segments. final Set refreshed = refreshSegments(segmentsToRefresh); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index 8aa6a36f8f84..5480cd61f352 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -26,11 +26,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.coordinator.CoordinatorClient; import org.apache.druid.client.coordinator.NoopCoordinatorClient; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.guava.Sequences; @@ -42,7 +45,9 @@ import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; +import org.apache.druid.segment.QueryableIndexStorageAdapter; import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinConditionAnalysis; import org.apache.druid.segment.join.Joinable; import org.apache.druid.segment.join.JoinableFactory; @@ -74,17 +79,17 @@ import org.junit.Test; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -// test polling from coordinator and when coordinator doesn't return anything -// test the result public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon { private final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); @@ -154,10 +159,10 @@ public void tearDown() throws Exception public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { - return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); + return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT, new NoopCoordinatorClient()); } - public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMetadataCacheConfig config) throws InterruptedException + public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMetadataCacheConfig config, CoordinatorClient coordinatorClient) throws InterruptedException { Preconditions.checkState(runningSchema == null); runningSchema = new BrokerSegmentMetadataCache( @@ -168,7 +173,7 @@ public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMeta new InternalQueryConfig(), new NoopServiceEmitter(), new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), - new NoopCoordinatorClient() + coordinatorClient ) { @Override @@ -215,8 +220,9 @@ public void markDataSourceAsNeedRebuild(String datasource) @Override @VisibleForTesting - public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws - IOException + public void refresh( + final Set segmentsToRefresh, + final Set dataSourcesToRebuild) throws IOException { super.refresh(segmentsToRefresh, dataSourcesToRebuild); refreshLatch.countDown(); @@ -229,17 +235,109 @@ public void refresh(final Set segmentsToRefresh, final Set da } @Test - public void testGetTableMap() throws InterruptedException + public void testGetAllDsSchemaFromCoordinator() throws InterruptedException { - BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema.getDatasourceNames()); + final RowSignature dataSource1RowSignature = new QueryableIndexStorageAdapter(index1).getRowSignature(); + final RowSignature dataSource2RowSignature = new QueryableIndexStorageAdapter(index2).getRowSignature(); + final RowSignature someDataSourceRowSignature = new QueryableIndexStorageAdapter(indexAuto1).getRowSignature(); + final RowSignature foo3RowSignature = new QueryableIndexStorageAdapter(indexAuto2).getRowSignature(); + + NoopCoordinatorClient coordinatorClient = new NoopCoordinatorClient() { + @Override + public ListenableFuture> fetchDataSourceInformation(Set datasources) + { + Map dataSourceInformationMap = new HashMap<>(); + dataSourceInformationMap.put(DATASOURCE1, new DataSourceInformation(DATASOURCE1, dataSource1RowSignature)); + dataSourceInformationMap.put(DATASOURCE2, new DataSourceInformation(DATASOURCE2, dataSource2RowSignature)); + dataSourceInformationMap.put(SOME_DATASOURCE, new DataSourceInformation(SOME_DATASOURCE, someDataSourceRowSignature)); + dataSourceInformationMap.put("foo3", new DataSourceInformation("foo3", foo3RowSignature)); + + return Futures.immediateFuture(new ArrayList<>(dataSourceInformationMap.values())); + } + }; + + QueryLifecycleFactory factoryMock = EasyMock.createMock(QueryLifecycleFactory.class); + + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + factoryMock, + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + coordinatorClient + ); + schema.start(); + schema.awaitInitialization(); final Set tableNames = schema.getDatasourceNames(); - Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), tableNames); + Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE, "foo3"), tableNames); + + Assert.assertEquals(dataSource1RowSignature, schema.getPhysicalDatasourceMetadata(DATASOURCE1).rowSignature()); + Assert.assertEquals(dataSource2RowSignature, schema.getPhysicalDatasourceMetadata(DATASOURCE2).rowSignature()); + Assert.assertEquals(someDataSourceRowSignature, schema.getPhysicalDatasourceMetadata(SOME_DATASOURCE).rowSignature()); + Assert.assertEquals(foo3RowSignature, schema.getPhysicalDatasourceMetadata("foo3").rowSignature()); + } + + @Test + public void testGetFewDsSchemaFromCoordinator() throws InterruptedException + { + final RowSignature dataSource1RowSignature = new QueryableIndexStorageAdapter(index1).getRowSignature(); + final RowSignature dataSource2RowSignature = new QueryableIndexStorageAdapter(index2).getRowSignature(); + final RowSignature someDataSourceRowSignature = new QueryableIndexStorageAdapter(indexAuto1).getRowSignature(); + + NoopCoordinatorClient coordinatorClient = new NoopCoordinatorClient() { + @Override + public ListenableFuture> fetchDataSourceInformation(Set datasources) + { + Map dataSourceInformationMap = new HashMap<>(); + dataSourceInformationMap.put(DATASOURCE1, new DataSourceInformation(DATASOURCE1, dataSource1RowSignature)); + dataSourceInformationMap.put(DATASOURCE2, new DataSourceInformation(DATASOURCE2, dataSource2RowSignature)); + dataSourceInformationMap.put(SOME_DATASOURCE, new DataSourceInformation(SOME_DATASOURCE, someDataSourceRowSignature)); + return Futures.immediateFuture(new ArrayList<>(dataSourceInformationMap.values())); + } + }; + + SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( + new TableDataSource("foo3"), + new MultipleSpecificSegmentSpec(Collections.singletonList(realtimeSegment1.getId().toDescriptor())), + new AllColumnIncluderator(), + false, + ImmutableMap.of(QueryContexts.BROKER_PARALLEL_MERGE_KEY, false), + EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), + false, + null, + null + ); + + QueryLifecycleFactory factoryMock = EasyMock.createMock(QueryLifecycleFactory.class); + QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); + EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); + EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) + .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); + + BrokerSegmentMetadataCache schema = new BrokerSegmentMetadataCache( + factoryMock, + serverView, + SEGMENT_CACHE_CONFIG_DEFAULT, + new NoopEscalator(), + new InternalQueryConfig(), + new NoopServiceEmitter(), + new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + coordinatorClient + ); + + EasyMock.replay(factoryMock, lifecycleMock); + + schema.start(); + schema.awaitInitialization(); + + EasyMock.verify(factoryMock, lifecycleMock); } @Test - public void testSchemaInit() throws InterruptedException + public void testGetTableMap() throws InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE), schema.getDatasourceNames()); @@ -308,7 +406,8 @@ public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePoli { return new SegmentMetadataCache.FirstTypeMergePolicy(); } - } + }, + new NoopCoordinatorClient() ); final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata(CalciteTests.SOME_DATASOURCE); final DruidTable table = new DatasourceTable(fooDs); @@ -569,11 +668,6 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw @Test public void testRunSegmentMetadataQueryWithContext() throws Exception { - Map queryContext = ImmutableMap.of( - QueryContexts.PRIORITY_KEY, 5, - QueryContexts.BROKER_PARALLEL_MERGE_KEY, false - ); - String brokerInternalQueryConfigJson = "{\"context\": { \"priority\": 5} }"; TestHelper.makeJsonMapper(); @@ -584,24 +678,6 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception InternalQueryConfig.class ); - DataSegment segment = newSegment("test", 0); - List segmentIterable = ImmutableList.of(segment.getId()); - - // This is the query that we expect this method to create. We will be testing that it matches the query generated by the method under test. - SegmentMetadataQuery expectedMetadataQuery = new SegmentMetadataQuery( - new TableDataSource(segment.getDataSource()), - new MultipleSpecificSegmentSpec( - segmentIterable.stream() - .map(SegmentId::toDescriptor).collect(Collectors.toList())), - new AllColumnIncluderator(), - false, - queryContext, - EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), - false, - null, - null - ); - QueryLifecycleFactory factoryMock = EasyMock.createMock(QueryLifecycleFactory.class); QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); @@ -617,17 +693,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception new NoopCoordinatorClient() ); - EasyMock.expect(factoryMock.factorize()).andReturn(lifecycleMock).once(); - // This is the mat of the test, making sure that the query created by the method under test matches the expected query, specifically the operator configured context - EasyMock.expect(lifecycleMock.runSimple(expectedMetadataQuery, AllowAllAuthenticator.ALLOW_ALL_RESULT, Access.OK)) - .andReturn(QueryResponse.withEmptyContext(Sequences.empty())); - - EasyMock.replay(factoryMock, lifecycleMock); - - mySchema.runSegmentMetadataQuery(segmentIterable); - - EasyMock.verify(factoryMock, lifecycleMock); - + checkRunSegmentMetadataQueryWithContext(mySchema, factoryMock, lifecycleMock); } @Test From eb1771ffe0f1d0c81463fa448985913f1fc9b769 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 14:45:35 +0530 Subject: [PATCH 24/36] fix checkstyle issue --- .../metadata/SegmentMetadataCache.java | 2 - .../metadata/SegmentMetadataCacheCommon.java | 125 +++++++++--------- .../metadata/SegmentMetadataCacheTest.java | 10 -- .../schema/BrokerSegmentMetadataCache.java | 1 - 4 files changed, 60 insertions(+), 78 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 6d80bd9390a8..e5709fac0dbd 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -61,9 +61,7 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.Types; -import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.QueryResponse; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.security.Access; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 096e96ed39e7..19540e231fe7 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -53,11 +53,7 @@ import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; -import org.apache.druid.segment.QueryableIndexStorageAdapter; -import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.incremental.IncrementalIndexSchema; -import org.apache.druid.segment.join.table.IndexedTable; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; @@ -84,7 +80,6 @@ import java.io.File; import java.io.IOException; import java.util.EnumSet; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -159,32 +154,32 @@ public void setupData() throws Exception { final File tmpDir = temporaryFolder.newFolder(); index1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(ROWS1) - .buildMMappedIndex(); + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(ROWS1) + .buildMMappedIndex(); index2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "2")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withMetrics(new LongSumAggregatorFactory("m1", "m1")) - .withRollup(false) - .build() - ) - .rows(ROWS2) - .buildMMappedIndex(); + .tmpDir(new File(tmpDir, "2")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withMetrics(new LongSumAggregatorFactory("m1", "m1")) + .withRollup(false) + .build() + ) + .rows(ROWS2) + .buildMMappedIndex(); final InputRowSchema rowSchema = new InputRowSchema( new TimestampSpec("t", null, null), @@ -219,44 +214,44 @@ public void setupData() throws Exception ); indexAuto1 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec(rowSchema.getTimestampSpec()) - .withDimensionsSpec(rowSchema.getDimensionsSpec()) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows1) - .buildMMappedIndex(); + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec(rowSchema.getTimestampSpec()) + .withDimensionsSpec(rowSchema.getDimensionsSpec()) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows1) + .buildMMappedIndex(); indexAuto2 = IndexBuilder.create() - .tmpDir(new File(tmpDir, "1")) - .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) - .schema( - new IncrementalIndexSchema.Builder() - .withTimestampSpec( - new TimestampSpec("t", null, null) - ) - .withDimensionsSpec( - DimensionsSpec.builder().useSchemaDiscovery(true).build() - ) - .withMetrics( - new CountAggregatorFactory("cnt"), - new DoubleSumAggregatorFactory("m1", "m1"), - new HyperUniquesAggregatorFactory("unique_dim1", "dim1") - ) - .withRollup(false) - .build() - ) - .rows(autoRows2) - .buildMMappedIndex(); + .tmpDir(new File(tmpDir, "1")) + .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) + .schema( + new IncrementalIndexSchema.Builder() + .withTimestampSpec( + new TimestampSpec("t", null, null) + ) + .withDimensionsSpec( + DimensionsSpec.builder().useSchemaDiscovery(true).build() + ) + .withMetrics( + new CountAggregatorFactory("cnt"), + new DoubleSumAggregatorFactory("m1", "m1"), + new HyperUniquesAggregatorFactory("unique_dim1", "dim1") + ) + .withRollup(false) + .build() + ) + .rows(autoRows2) + .buildMMappedIndex(); walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( DataSegment.builder() diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index 4c969464a6ab..c60d08f80bc6 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -28,26 +28,17 @@ import org.apache.druid.client.ImmutableDruidServer; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.java.util.metrics.StubServiceEmitter; -import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; -import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.QueryResponse; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.server.metrics.NoopServiceEmitter; -import org.apache.druid.server.security.Access; -import org.apache.druid.server.security.AllowAllAuthenticator; import org.apache.druid.server.security.NoopEscalator; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; @@ -58,7 +49,6 @@ import org.junit.Test; import java.io.IOException; -import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 921bbcf85461..f59358586cde 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -38,7 +38,6 @@ import java.io.IOException; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; From 1440dacec90f31546d04a44bc326469cc84df464 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 17:57:49 +0530 Subject: [PATCH 25/36] Add test for QueryableCoordinatorServerView --- .../client/CoordinatorServerViewTest.java | 373 ------------------ ...Test.java => CoordinatorTimelineTest.java} | 132 +++++-- 2 files changed, 90 insertions(+), 415 deletions(-) delete mode 100644 server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java rename server/src/test/java/org/apache/druid/client/{QueryableCoordinatorServerViewTest.java => CoordinatorTimelineTest.java} (81%) diff --git a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java deleted file mode 100644 index 1bca2a0690af..000000000000 --- a/server/src/test/java/org/apache/druid/client/CoordinatorServerViewTest.java +++ /dev/null @@ -1,373 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.client; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Function; -import com.google.common.base.Predicates; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import org.apache.curator.utils.ZKPaths; -import org.apache.druid.curator.CuratorTestBase; -import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.segment.TestHelper; -import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.server.coordination.ServerType; -import org.apache.druid.server.initialization.ZkPathsConfig; -import org.apache.druid.server.metrics.NoopServiceEmitter; -import org.apache.druid.timeline.DataSegment; -import org.apache.druid.timeline.TimelineLookup; -import org.apache.druid.timeline.TimelineObjectHolder; -import org.apache.druid.timeline.partition.NoneShardSpec; -import org.apache.druid.timeline.partition.PartitionHolder; -import org.joda.time.Interval; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; - -public class CoordinatorServerViewTest extends CuratorTestBase -{ - private final ObjectMapper jsonMapper; - private final ZkPathsConfig zkPathsConfig; - private final String inventoryPath; - - private CountDownLatch segmentViewInitLatch; - private CountDownLatch segmentAddedLatch; - private CountDownLatch segmentRemovedLatch; - - private BatchServerInventoryView baseView; - private CoordinatorServerView overlordServerView; - - public CoordinatorServerViewTest() - { - jsonMapper = TestHelper.makeJsonMapper(); - zkPathsConfig = new ZkPathsConfig(); - inventoryPath = zkPathsConfig.getLiveSegmentsPath(); - } - - @Before - public void setUp() throws Exception - { - setupServerAndCurator(); - curator.start(); - curator.blockUntilConnected(); - } - - @Test - public void testSingleServerAddedRemovedSegment() throws Exception - { - segmentViewInitLatch = new CountDownLatch(1); - segmentAddedLatch = new CountDownLatch(1); - segmentRemovedLatch = new CountDownLatch(1); - - setupViews(); - - final DruidServer druidServer = new DruidServer( - "localhost:1234", - "localhost:1234", - null, - 10000000L, - ServerType.HISTORICAL, - "default_tier", - 0 - ); - - setupZNodeForServer(druidServer, zkPathsConfig, jsonMapper); - - final DataSegment segment = dataSegmentWithIntervalAndVersion("2014-10-20T00:00:00Z/P1D", "v1"); - final int partition = segment.getShardSpec().getPartitionNum(); - final Interval intervals = Intervals.of("2014-10-20T00:00:00Z/P1D"); - announceSegmentForServer(druidServer, segment, zkPathsConfig, jsonMapper); - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); - - TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); - List serverLookupRes = (List) timeline.lookup( - intervals - ); - Assert.assertEquals(1, serverLookupRes.size()); - - TimelineObjectHolder actualTimelineObjectHolder = serverLookupRes.get(0); - Assert.assertEquals(intervals, actualTimelineObjectHolder.getInterval()); - Assert.assertEquals("v1", actualTimelineObjectHolder.getVersion()); - - PartitionHolder actualPartitionHolder = actualTimelineObjectHolder.getObject(); - Assert.assertTrue(actualPartitionHolder.isComplete()); - Assert.assertEquals(1, Iterables.size(actualPartitionHolder)); - - SegmentLoadInfo segmentLoadInfo = actualPartitionHolder.iterator().next().getObject(); - Assert.assertFalse(segmentLoadInfo.isEmpty()); - Assert.assertEquals( - druidServer.getMetadata(), - Iterables.getOnlyElement(segmentLoadInfo.toImmutableSegmentLoadInfo().getServers()) - ); - Assert.assertNotNull(timeline.findChunk(intervals, "v1", partition)); - - unannounceSegmentForServer(druidServer, segment); - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); - - Assert.assertEquals( - 0, - ((List) timeline.lookup(Intervals.of("2014-10-20T00:00:00Z/P1D"))).size() - ); - Assert.assertNull(timeline.findChunk(intervals, "v1", partition)); - } - - @Test - public void testMultipleServerAddedRemovedSegment() throws Exception - { - segmentViewInitLatch = new CountDownLatch(1); - segmentAddedLatch = new CountDownLatch(5); - - // temporarily set latch count to 1 - segmentRemovedLatch = new CountDownLatch(1); - - setupViews(); - - final List druidServers = Lists.transform( - ImmutableList.of("localhost:0", "localhost:1", "localhost:2", "localhost:3", "localhost:4"), - new Function() - { - @Override - public DruidServer apply(String input) - { - return new DruidServer( - input, - input, - null, - 10000000L, - ServerType.HISTORICAL, - "default_tier", - 0 - ); - } - } - ); - - for (DruidServer druidServer : druidServers) { - setupZNodeForServer(druidServer, zkPathsConfig, jsonMapper); - } - - final List segments = Lists.transform( - ImmutableList.of( - Pair.of("2011-04-01/2011-04-03", "v1"), - Pair.of("2011-04-03/2011-04-06", "v1"), - Pair.of("2011-04-01/2011-04-09", "v2"), - Pair.of("2011-04-06/2011-04-09", "v3"), - Pair.of("2011-04-01/2011-04-02", "v3") - ), new Function, DataSegment>() - { - @Override - public DataSegment apply(Pair input) - { - return dataSegmentWithIntervalAndVersion(input.lhs, input.rhs); - } - } - ); - - for (int i = 0; i < 5; ++i) { - announceSegmentForServer(druidServers.get(i), segments.get(i), zkPathsConfig, jsonMapper); - } - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); - - TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); - assertValues( - Arrays.asList( - createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), - createExpected("2011-04-02/2011-04-06", "v2", druidServers.get(2), segments.get(2)), - createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3)) - ), - (List) timeline.lookup( - Intervals.of( - "2011-04-01/2011-04-09" - ) - ) - ); - - // unannounce the segment created by dataSegmentWithIntervalAndVersion("2011-04-01/2011-04-09", "v2") - unannounceSegmentForServer(druidServers.get(2), segments.get(2)); - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); - - // renew segmentRemovedLatch since we still have 4 segments to unannounce - segmentRemovedLatch = new CountDownLatch(4); - - timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); - assertValues( - Arrays.asList( - createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), - createExpected("2011-04-02/2011-04-03", "v1", druidServers.get(0), segments.get(0)), - createExpected("2011-04-03/2011-04-06", "v1", druidServers.get(1), segments.get(1)), - createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3)) - ), - (List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09")) - ); - - // unannounce all the segments - for (int i = 0; i < 5; ++i) { - // skip the one that was previously unannounced - if (i != 2) { - unannounceSegmentForServer(druidServers.get(i), segments.get(i)); - } - } - Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); - - Assert.assertEquals( - 0, - ((List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09"))).size() - ); - } - - private void unannounceSegmentForServer(DruidServer druidServer, DataSegment segment) throws Exception - { - curator - .delete() - .guaranteed() - .forPath(ZKPaths.makePath(inventoryPath, druidServer.getHost(), segment.getId().toString())); - } - - private Pair>> createExpected( - String intervalStr, - String version, - DruidServer druidServer, - DataSegment segment - ) - { - return Pair.of(Intervals.of(intervalStr), Pair.of(version, Pair.of(druidServer, segment))); - } - - private void assertValues( - List>>> expected, List actual - ) - { - Assert.assertEquals(expected.size(), actual.size()); - - for (int i = 0; i < expected.size(); ++i) { - Pair>> expectedPair = expected.get(i); - TimelineObjectHolder actualTimelineObjectHolder = actual.get(i); - - Assert.assertEquals(expectedPair.lhs, actualTimelineObjectHolder.getInterval()); - Assert.assertEquals(expectedPair.rhs.lhs, actualTimelineObjectHolder.getVersion()); - - PartitionHolder actualPartitionHolder = actualTimelineObjectHolder.getObject(); - Assert.assertTrue(actualPartitionHolder.isComplete()); - Assert.assertEquals(1, Iterables.size(actualPartitionHolder)); - - SegmentLoadInfo segmentLoadInfo = actualPartitionHolder.iterator().next().getObject(); - Assert.assertFalse(segmentLoadInfo.isEmpty()); - Assert.assertEquals(expectedPair.rhs.rhs.lhs.getMetadata(), - Iterables.getOnlyElement(segmentLoadInfo.toImmutableSegmentLoadInfo().getServers())); - } - } - - private void setupViews() throws Exception - { - baseView = new BatchServerInventoryView( - zkPathsConfig, - curator, - jsonMapper, - Predicates.alwaysTrue(), - "test" - ) - { - @Override - public void registerSegmentCallback(Executor exec, final SegmentCallback callback) - { - super.registerSegmentCallback( - exec, - new SegmentCallback() - { - @Override - public CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment) - { - CallbackAction res = callback.segmentAdded(server, segment); - segmentAddedLatch.countDown(); - return res; - } - - @Override - public CallbackAction segmentRemoved(DruidServerMetadata server, DataSegment segment) - { - CallbackAction res = callback.segmentRemoved(server, segment); - segmentRemovedLatch.countDown(); - return res; - } - - @Override - public CallbackAction segmentViewInitialized() - { - CallbackAction res = callback.segmentViewInitialized(); - segmentViewInitLatch.countDown(); - return res; - } - } - ); - } - }; - - overlordServerView = new CoordinatorServerView( - baseView, - new CoordinatorSegmentWatcherConfig(), - new NoopServiceEmitter() - ); - - baseView.start(); - overlordServerView.start(); - } - - private DataSegment dataSegmentWithIntervalAndVersion(String intervalStr, String version) - { - return DataSegment.builder() - .dataSource("test_overlord_server_view") - .interval(Intervals.of(intervalStr)) - .loadSpec( - ImmutableMap.of( - "type", - "local", - "path", - "somewhere" - ) - ) - .version(version) - .dimensions(ImmutableList.of()) - .metrics(ImmutableList.of()) - .shardSpec(NoneShardSpec.instance()) - .binaryVersion(9) - .size(0) - .build(); - } - - @After - public void tearDown() throws Exception - { - baseView.stop(); - tearDownServerAndCurator(); - } -} diff --git a/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java b/server/src/test/java/org/apache/druid/client/CoordinatorTimelineTest.java similarity index 81% rename from server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java rename to server/src/test/java/org/apache/druid/client/CoordinatorTimelineTest.java index fccbd1651df8..402113e051d4 100644 --- a/server/src/test/java/org/apache/druid/client/QueryableCoordinatorServerViewTest.java +++ b/server/src/test/java/org/apache/druid/client/CoordinatorTimelineTest.java @@ -20,8 +20,6 @@ package org.apache.druid.client; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.smile.SmileFactory; -import com.fasterxml.jackson.dataformat.smile.SmileGenerator; import com.google.common.base.Function; import com.google.common.base.Predicates; import com.google.common.collect.ImmutableList; @@ -32,7 +30,6 @@ import org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy; import org.apache.druid.client.selector.RandomServerSelectorStrategy; import org.apache.druid.curator.CuratorTestBase; -import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.http.client.HttpClient; @@ -61,20 +58,18 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; -public class QueryableCoordinatorServerViewTest extends CuratorTestBase +public class CoordinatorTimelineTest extends CuratorTestBase { private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; private final String inventoryPath; - private CountDownLatch segmentViewInitLatch; private CountDownLatch segmentAddedLatch; private CountDownLatch segmentRemovedLatch; private BatchServerInventoryView baseView; - private QueryableCoordinatorServerView overlordServerView; - public QueryableCoordinatorServerViewTest() + public CoordinatorTimelineTest() { jsonMapper = TestHelper.makeJsonMapper(); zkPathsConfig = new ZkPathsConfig(); @@ -90,7 +85,7 @@ public void setUp() throws Exception } @Test - public void testSingleServerAddedRemovedSegment() throws Exception + public void testSingleServerAddedRemovedSegment_CoordinatorServerView() throws Exception { segmentViewInitLatch = new CountDownLatch(1); segmentAddedLatch = new CountDownLatch(1); @@ -98,6 +93,46 @@ public void testSingleServerAddedRemovedSegment() throws Exception setupViews(); + CoordinatorServerView coordinatorServerView = new CoordinatorServerView( + baseView, + new CoordinatorSegmentWatcherConfig(), + new NoopServiceEmitter() + ); + + baseView.start(); + coordinatorServerView.start(); + + testSingleServerAddedRemovedSegment(coordinatorServerView); + } + + @Test + public void testSingleServerAddedRemovedSegment_QueryableCoordinatorServerView() throws Exception + { + segmentViewInitLatch = new CountDownLatch(1); + segmentAddedLatch = new CountDownLatch(1); + segmentRemovedLatch = new CountDownLatch(1); + + setupViews(); + + QueryableCoordinatorServerView queryableCoordinatorServerView = new QueryableCoordinatorServerView( + EasyMock.createMock(QueryToolChestWarehouse.class), + EasyMock.createMock(QueryWatcher.class), + new ObjectMapper(), + EasyMock.createMock(HttpClient.class), + baseView, + new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), + new NoopServiceEmitter(), + new CoordinatorSegmentWatcherConfig() + ); + + baseView.start(); + queryableCoordinatorServerView.start(); + + testSingleServerAddedRemovedSegment(queryableCoordinatorServerView); + } + + private void testSingleServerAddedRemovedSegment(CoordinatorTimeline coordinatorTimeline) throws Exception + { final DruidServer druidServer = new DruidServer( "localhost:1234", "localhost:1234", @@ -117,7 +152,7 @@ public void testSingleServerAddedRemovedSegment() throws Exception Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); - TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + TimelineLookup timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); List serverLookupRes = (List) timeline.lookup( intervals ); @@ -142,7 +177,7 @@ public void testSingleServerAddedRemovedSegment() throws Exception unannounceSegmentForServer(druidServer, segment); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); - timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); Assert.assertEquals( 0, ((List) timeline.lookup(Intervals.of("2014-10-20T00:00:00Z/P1D"))).size() @@ -151,16 +186,54 @@ public void testSingleServerAddedRemovedSegment() throws Exception } @Test - public void testMultipleServerAddedRemovedSegment() throws Exception + public void testMultipleServerAddedRemovedSegment_CoordinatorServerView() throws Exception { segmentViewInitLatch = new CountDownLatch(1); segmentAddedLatch = new CountDownLatch(5); + segmentRemovedLatch = new CountDownLatch(1); + + setupViews(); + + CoordinatorServerView coordinatorServerView = new CoordinatorServerView( + baseView, + new CoordinatorSegmentWatcherConfig(), + new NoopServiceEmitter() + ); + + baseView.start(); + coordinatorServerView.start(); - // temporarily set latch count to 1 + testMultipleServerAddedRemovedSegment(coordinatorServerView); + } + + @Test + public void testMultipleServerAddedRemovedSegment_QueryableCoordinatorServerView() throws Exception + { + segmentViewInitLatch = new CountDownLatch(1); + segmentAddedLatch = new CountDownLatch(5); segmentRemovedLatch = new CountDownLatch(1); setupViews(); + QueryableCoordinatorServerView queryableCoordinatorServerView = new QueryableCoordinatorServerView( + EasyMock.createMock(QueryToolChestWarehouse.class), + EasyMock.createMock(QueryWatcher.class), + new ObjectMapper(), + EasyMock.createMock(HttpClient.class), + baseView, + new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), + new NoopServiceEmitter(), + new CoordinatorSegmentWatcherConfig() + ); + + baseView.start(); + queryableCoordinatorServerView.start(); + + testMultipleServerAddedRemovedSegment(queryableCoordinatorServerView); + } + + void testMultipleServerAddedRemovedSegment(CoordinatorTimeline coordinatorTimeline) throws Exception + { final List druidServers = Lists.transform( ImmutableList.of("localhost:0", "localhost:1", "localhost:2", "localhost:3", "localhost:4"), new Function() @@ -208,7 +281,7 @@ public DataSegment apply(Pair input) Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch)); Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch)); - TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + TimelineLookup timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); assertValues( Arrays.asList( createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), @@ -229,7 +302,8 @@ public DataSegment apply(Pair input) // renew segmentRemovedLatch since we still have 4 segments to unannounce segmentRemovedLatch = new CountDownLatch(4); - timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); + + timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); assertValues( Arrays.asList( createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), @@ -249,8 +323,7 @@ public DataSegment apply(Pair input) } Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch)); - timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view")); - + timeline = coordinatorTimeline.getTimeline(new TableDataSource("test_overlord_server_view")); Assert.assertEquals( 0, ((List) timeline.lookup(Intervals.of("2011-04-01/2011-04-09"))).size() @@ -299,7 +372,7 @@ private void assertValues( } } - private void setupViews() throws Exception + private void setupViews() { baseView = new BatchServerInventoryView( zkPathsConfig, @@ -343,33 +416,8 @@ public CallbackAction segmentViewInitialized() ); } }; - - overlordServerView = new QueryableCoordinatorServerView( - EasyMock.createMock(QueryToolChestWarehouse.class), - EasyMock.createMock(QueryWatcher.class), - getSmileMapper(), - EasyMock.createMock(HttpClient.class), - baseView, - new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()), - new NoopServiceEmitter(), - new CoordinatorSegmentWatcherConfig() - ); - - baseView.start(); - overlordServerView.start(); } - private ObjectMapper getSmileMapper() - { - final SmileFactory smileFactory = new SmileFactory(); - smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false); - smileFactory.delegateToTextual(true); - final ObjectMapper retVal = new DefaultObjectMapper(smileFactory, "broker"); - retVal.getFactory().setCodec(retVal); - return retVal; - } - - private DataSegment dataSegmentWithIntervalAndVersion(String intervalStr, String version) { return DataSegment.builder() From 10068b60a8f143aaebf67d86d17066dbef1e03ee Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 18:10:44 +0530 Subject: [PATCH 26/36] Fix SegmentStatusInClusterTest --- .../apache/druid/timeline/SegmentStatusInClusterTest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java b/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java index 54190870e77c..82b77ddee2bc 100644 --- a/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java +++ b/processing/src/test/java/org/apache/druid/timeline/SegmentStatusInClusterTest.java @@ -49,6 +49,8 @@ public class SegmentStatusInClusterTest private static final ImmutableMap LOAD_SPEC = ImmutableMap.of("something", "or_other"); private static final boolean OVERSHADOWED = true; private static final Integer REPLICATION_FACTOR = 2; + private static final Long NUM_ROWS = 10L; + private static final boolean REALTIME = true; private static final int TEST_VERSION = 0x9; private static final SegmentStatusInCluster SEGMENT = createSegmentForTest(); @@ -76,7 +78,7 @@ private static SegmentStatusInCluster createSegmentForTest() 1 ); - return new SegmentStatusInCluster(dataSegment, OVERSHADOWED, REPLICATION_FACTOR, 10L, true); + return new SegmentStatusInCluster(dataSegment, OVERSHADOWED, REPLICATION_FACTOR, NUM_ROWS, REALTIME); } @Test @@ -87,7 +89,7 @@ public void testUnwrappedSegmentWithOvershadowedStatusDeserialization() throws E JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); - Assert.assertEquals(12, objectMap.size()); + Assert.assertEquals(14, objectMap.size()); Assert.assertEquals("something", objectMap.get("dataSource")); Assert.assertEquals(INTERVAL.toString(), objectMap.get("interval")); Assert.assertEquals("1", objectMap.get("version")); @@ -99,6 +101,8 @@ public void testUnwrappedSegmentWithOvershadowedStatusDeserialization() throws E Assert.assertEquals(1, objectMap.get("size")); Assert.assertEquals(OVERSHADOWED, objectMap.get("overshadowed")); Assert.assertEquals(REPLICATION_FACTOR, objectMap.get("replicationFactor")); + Assert.assertEquals(NUM_ROWS.intValue(), objectMap.get("numRows")); + Assert.assertEquals(REALTIME, objectMap.get("realtime")); final String json = MAPPER.writeValueAsString(SEGMENT); From 032734af9394b634f0dd027b3a22cffaf68a1fea Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 18:15:18 +0530 Subject: [PATCH 27/36] Address intellij inspection --- .../druid/segment/metadata/SegmentMetadataCacheCommon.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 19540e231fe7..a22eb4c71d99 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -423,7 +423,7 @@ public void checkNullDatasource(SegmentMetadataCache schema) throws IOException Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); } - public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) throws InterruptedException + public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) { Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); final List segments = segmentsMetadata.values() From 80f442400e9a712f59a39a6330fc6f1f5309c195 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 18:27:33 +0530 Subject: [PATCH 28/36] Add undeclared dependency in server module --- server/pom.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/pom.xml b/server/pom.xml index 5ba0b170a9ed..16c0399c5dbc 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -49,6 +49,11 @@ ${project.parent.version} runtime + + com.google.http-client + google-http-client + runtime + jakarta.inject @@ -307,10 +312,10 @@ com.fasterxml.jackson.module jackson-module-guice - + org.apache.commons commons-lang3 - + From 33a8dd54d3375d62ce83a284b32a55262b538919 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Tue, 12 Sep 2023 21:37:07 +0530 Subject: [PATCH 29/36] Remove enabled field from SegmentMetadataCacheConfig --- .../segment/metadata/SegmentMetadataCacheConfig.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index 2ebcb0e53c0a..0feac95506d0 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -27,11 +27,8 @@ public class SegmentMetadataCacheConfig { @JsonProperty private boolean awaitInitializationOnStart = false; - @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); - @JsonProperty - private final boolean enabled = false; @JsonProperty private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = @@ -71,9 +68,4 @@ public Period getMetadataRefreshPeriod() { return metadataRefreshPeriod; } - - public boolean isEnabled() - { - return enabled; - } } From 7a7ca55298f1ad283610afb018adcbc8564f087d Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 13 Sep 2023 14:58:12 +0530 Subject: [PATCH 30/36] Add class to manage druid table information in SegmentMetadataCache, add javadocs for classes --- .../QueryableCoordinatorServerView.java | 13 +-- .../client/coordinator/CoordinatorClient.java | 2 +- .../druid/client/selector/ServerSelector.java | 4 +- .../metadata/DataSourceInformation.java | 2 +- .../metadata/SegmentMetadataCache.java | 84 +++++++++++++------ .../metadata/SegmentMetadataCacheConfig.java | 6 ++ .../druid/server/http/MetadataResource.java | 8 +- .../metadata/SegmentMetadataCacheCommon.java | 12 --- .../metadata/SegmentMetadataCacheTest.java | 8 +- .../schema/BrokerSegmentMetadataCache.java | 68 ++++++++++----- .../BrokerSegmentMetadataCacheConfig.java | 14 ++++ .../calcite/schema/MetadataSegmentView.java | 2 +- .../PhysicalDatasourceMetadataBuilder.java | 20 +++-- .../BrokerSegmentMetadataCacheTest.java | 15 +++- 14 files changed, 178 insertions(+), 80 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index 719562a73716..0f69f2699a7f 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -44,11 +44,12 @@ /** * ServerView of coordinator for the state of segments being loaded in the cluster. *
    - * This class simply extends {@link BrokerServerView} and implements methods from {@link CoordinatorTimeline} - * for backward compatibility. This newer implementation is primarily required by - * {@link org.apache.druid.segment.metadata.SegmentMetadataCache} which will run on the Coordinator. + * This class extends {@link BrokerServerView} and implements {@link CoordinatorTimeline}. + * The main distinction between this class and {@link CoordinatorServerView} is the maintenance of a timeline + * of {@link ServerSelector} objects, while the other class stores {@link SegmentLoadInfo} object in its timeline. *
    - * Once this class is stable {@link CoordinatorServerView} should be removed. + * A new timeline class (implementing {@link TimelineServerView}) is required for + * {@link org.apache.druid.segment.metadata.SegmentMetadataCache}, which will run on the Coordinator. */ @ManageLifecycle public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline @@ -78,8 +79,8 @@ public boolean isAwaitInitializationOnStart() } /** - * Internally this class maintains a timeline of {@link ServerSelector}. - * This method returns a newline of the object {@link SegmentLoadInfo}. + * This class maintains a timeline of {@link ServerSelector} objects. + * This method returns a new timeline of the object {@link SegmentLoadInfo}. * * @param dataSource dataSoruce * @return timeline for the given dataSource diff --git a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java index 2f51ca2ec743..ac61176d50fa 100644 --- a/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java +++ b/server/src/main/java/org/apache/druid/client/coordinator/CoordinatorClient.java @@ -48,7 +48,7 @@ public interface CoordinatorClient ListenableFuture> fetchUsedSegments(String dataSource, List intervals); /** - * Fetches information for the given dataSources. + * Retrieves detailed metadata information for the specified data sources, which includes {@code RowSignature}. */ ListenableFuture> fetchDataSourceInformation(Set datasources); diff --git a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java index 0050edfa8f81..616778987428 100644 --- a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java @@ -218,8 +218,8 @@ public boolean hasData() } /** - * This conversion is required to make the newer {@link org.apache.druid.client.QueryableCoordinatorServerView} - * implement methods from {@link org.apache.druid.client.CoordinatorTimeline} + * This conversion, allows {@link org.apache.druid.client.QueryableCoordinatorServerView} + * to implement methods from {@link org.apache.druid.client.CoordinatorTimeline}. * * @return {@link SegmentLoadInfo} */ diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index b8e8c7d5389c..1fe6534d923c 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -26,7 +26,7 @@ import java.util.Objects; /** - * Encapsulates information of a dataSource like schema. + * Encapsulates information about a dataSource, such as its schema. */ public class DataSourceInformation { diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index b1ceed6ca0f8..376d4582d5d4 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -92,10 +92,8 @@ import java.util.stream.StreamSupport; /** - * Broker-side cache of segment metadata which combines segments to identify - * datasources which become "tables" in Calcite. This cache provides the "physical" - * metadata about a datasource which is blended with catalog "logical" metadata - * to provide the final user-view of each datasource. + * Coordinator-side cache of segment metadata that combines segments to identify + * datasources.The cache provides metadata about a dataSource, see {@link DataSourceInformation}. */ @ManageLifecycle public class SegmentMetadataCache @@ -121,10 +119,11 @@ public class SegmentMetadataCache private final ColumnTypeMergePolicy columnTypeMergePolicy; /** - * Map of DataSource -> DruidTable. - * This map can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. + * Manages tables of DataSourceInformation. This manager is used to retrieve and store + * information related to dataSources. + * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final TableManager tableManager = new TableManager<>(); /** * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. @@ -281,16 +280,6 @@ public ServerView.CallbackAction serverSegmentRemoved( ); } - protected void removeFromTable(String s) - { - tables.remove(s); - } - - protected boolean tablesContains(String s) - { - return tables.containsKey(s); - } - private void startCacheExec() { cacheExec.submit( @@ -426,15 +415,15 @@ public void refresh(final Set segmentsToRefresh, final Set da } } - public void rebuildDatasource(String dataSource) + protected void rebuildDatasource(String dataSource) { final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tables.remove(dataSource); + tableManager.removeFromTable(dataSource); return; } - final DataSourceInformation oldTable = tables.put(dataSource, druidTable); + final DataSourceInformation oldTable = tableManager.put(dataSource, druidTable); if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { @@ -456,17 +445,17 @@ public void awaitInitialization() throws InterruptedException public DataSourceInformation getDatasource(String name) { - return tables.get(name); + return tableManager.get(name); } public Map getDataSourceInformationMap() { - return ImmutableMap.copyOf(tables); + return tableManager.getAll(); } public Set getDatasourceNames() { - return tables.keySet(); + return tableManager.getKeySet(); } @VisibleForTesting @@ -533,7 +522,7 @@ public void addSegment(final DruidServerMetadata server, final DataSegment segme } ); } - if (!tablesContains(segment.getDataSource())) { + if (!tableManager.tablesContains(segment.getDataSource())) { refreshImmediately = true; } @@ -564,7 +553,7 @@ public void removeSegment(final DataSegment segment) totalSegments--; } if (segmentsMap.isEmpty()) { - removeFromTable(segment.getDataSource()); + tableManager.removeFromTable(segment.getDataSource()); log.info("dataSource [%s] no longer exists, all metadata removed.", segment.getDataSource()); return null; } else { @@ -1127,4 +1116,49 @@ public String toString() return NAME; } } + + /** + * A generic class for managing table. + * + * @param The type of data associated with the tables (e.g., DataSourceInformation, PhysicalDataSourceMetadata). + */ + public static class TableManager + { + private final ConcurrentMap tables = new ConcurrentHashMap<>(); + + public void removeFromTable(String s) + { + tables.remove(s); + } + + public boolean tablesContains(String s) + { + return tables.containsKey(s); + } + + public T get(String s) + { + return tables.get(s); + } + + public Set getKeySet() + { + return tables.keySet(); + } + + public Map getAll() + { + return ImmutableMap.copyOf(tables); + } + + public T put(String key, T value) + { + return tables.put(key, value); + } + + public void putAll(Map anotherTable) + { + tables.putAll(anotherTable); + } + } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index 0feac95506d0..de11d2b491e3 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -23,10 +23,16 @@ import com.google.common.annotations.VisibleForTesting; import org.joda.time.Period; +/** + * Coordinator-side configuration class for customizing properties related to the SegmentMetadata cache. + */ public class SegmentMetadataCacheConfig { + // A flag indicating whether to wait for cache initialization during startup. @JsonProperty private boolean awaitInitializationOnStart = false; + + // Cache refresh interval. @JsonProperty private Period metadataRefreshPeriod = new Period("PT1M"); diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 447fac8784aa..c79962c0d111 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -28,7 +28,6 @@ import org.apache.druid.client.ImmutableDruidDataSource; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.Segments; -import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.metadata.DataSourceInformation; @@ -72,7 +71,6 @@ @Path("/druid/coordinator/v1/metadata") public class MetadataResource { - private final Logger log = new Logger(MetadataResource.class); private final SegmentsMetadataManager segmentsMetadataManager; private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; @@ -221,11 +219,13 @@ private Response getAllUsedSegmentsWithAdditionalDetails( replicationFactor, numRows, // published segment can't be realtime - false); + false + ); }); Stream finalSegments = segmentStatus; + // conditionally add realtime segments information if (null != includeRealtimeSegments && null != segmentMetadataCache) { final Stream realtimeSegmentStatus = segmentMetadataCache .getSegmentMetadataSnapshot() @@ -357,7 +357,7 @@ public Response getSegment( } /** - * Return schema for the given dataSources. + * API to fetch {@link DataSourceInformation} for the specified dataSources. */ @POST @Path("/dataSourceInformation") diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index a22eb4c71d99..5717ab476c6f 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -80,10 +80,8 @@ import java.io.File; import java.io.IOException; import java.util.EnumSet; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -347,16 +345,6 @@ QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) ); } - public void checkStaleDatasourceRefresh(SegmentMetadataCache schema) throws IOException - { - Set segments = new HashSet<>(); - Set datasources = new HashSet<>(); - datasources.add("wat"); - Assert.assertNull(schema.getDatasource("wat")); - schema.refresh(segments, datasources); - Assert.assertNull(schema.getDatasource("wat")); - } - public void checkRefreshShouldEmitMetrics( SegmentMetadataCache schema, String dataSource, diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index e69555ee96b9..cd664f13504e 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -49,6 +49,7 @@ import org.junit.Test; import java.io.IOException; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -905,7 +906,12 @@ public void testSegmentMetadataFallbackType() public void testStaleDatasourceRefresh() throws IOException, InterruptedException { SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - checkStaleDatasourceRefresh(schema); + Set segments = new HashSet<>(); + Set datasources = new HashSet<>(); + datasources.add("wat"); + Assert.assertNull(schema.getDatasource("wat")); + schema.refresh(segments, datasources); + Assert.assertNull(schema.getDatasource("wat")); } @Test diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index f59358586cde..90fdec9c54eb 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -21,6 +21,7 @@ import com.google.common.collect.Sets; import com.google.inject.Inject; +import io.vavr.Predicates; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.CoordinatorClient; @@ -32,7 +33,7 @@ import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.security.Escalator; -import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; import org.apache.druid.timeline.SegmentId; import java.io.IOException; @@ -40,20 +41,34 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; /** - * + * Broker-side cache of segment metadata that combines segments to identify + * datasources which become "tables" in Calcite. This cache provides the "physical" + * metadata about a datasource which is blended with catalog "logical" metadata + * to provide the final user-view of each datasource. + *

    + * This class extends {@link SegmentMetadataCache} and introduces following changes, + *

      + *
    • The refresh mechanism now includes polling the coordinator for dataSource schema, + * and falling back to running {@link org.apache.druid.query.metadata.metadata.SegmentMetadataQuery}.
      • + *
    • It builds and caches {@link PhysicalDatasourceMetadata} object as table + * schema instead of {@link DataSourceInformation}.
      • + *
    */ @ManageLifecycle public class BrokerSegmentMetadataCache extends SegmentMetadataCache { private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataCache.class); - private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; - private final ConcurrentMap tables = new ConcurrentHashMap<>(); + private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; + /** + * Manages tables of PhysicalDataSourceMetadata. This manager is used to retrieve and store + * information related to dataSources. + * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. + */ + private final TableManager tableManager = new TableManager<>(); private final CoordinatorClient coordinatorClient; @Inject @@ -80,6 +95,11 @@ public BrokerSegmentMetadataCache( this.coordinatorClient = coordinatorClient; } + /** + * Refreshes the set of segments in two steps: + * 1. Polls the coordinator for the dataSource schema to update the {@code tables}. + * 2. Refreshes the remaining set of segments by executing a SegmentMetadataQuery. + */ @Override public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException { @@ -87,7 +107,7 @@ public void refresh(final Set segmentsToRefresh, final Set da segmentsToRefresh.forEach(segment -> dataSourcesToQuery.add(segment.getDataSource())); - Map polledDataSourceMetadata = new HashMap<>(); + Map polledDataSourceMetadata = new HashMap<>(); // Fetch dataSource information from the Coordinator try { @@ -101,7 +121,10 @@ public void refresh(final Set segmentsToRefresh, final Set da log.warn(e, "Exception querying coordinator to fetch dataSourceInformation."); } - tables.putAll(polledDataSourceMetadata); + // remove any extra dataSources returned + polledDataSourceMetadata.keySet().removeIf(Predicates.not(dataSourcesToQuery::contains)); + + tableManager.putAll(polledDataSourceMetadata); // Remove segments of the dataSource from refresh list for which we received schema from the Coordinator. segmentsToRefresh.removeIf(segmentId -> polledDataSourceMetadata.containsKey(segmentId.getDataSource())); @@ -128,44 +151,47 @@ public void refresh(final Set segmentsToRefresh, final Set da } } + /** + * Build table schema and convert DataSourceInformation to PhysicalDatasourceMetadata. + */ @Override - public void rebuildDatasource(String dataSource) + protected void rebuildDatasource(String dataSource) { final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tables.remove(dataSource); + tableManager.removeFromTable(dataSource); return; } - final DatasourceTable.PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); - final DatasourceTable.PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); + final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); + final PhysicalDatasourceMetadata oldTable = tableManager.put(dataSource, physicalDatasourceMetadata); if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { - log.info("[%s] signature is unchanged.", dataSource); + log.debug("[%s] signature is unchanged.", dataSource); } } @Override public Set getDatasourceNames() { - return tables.keySet(); + return tableManager.getKeySet(); } - @Override - protected void removeFromTable(String s) + public PhysicalDatasourceMetadata getPhysicalDatasourceMetadata(String name) { - tables.remove(s); + return tableManager.get(name); } @Override - protected boolean tablesContains(String s) + public DataSourceInformation getDatasource(String name) { - return tables.containsKey(s); + throw new UnsupportedOperationException(); } - public DatasourceTable.PhysicalDatasourceMetadata getPhysicalDatasourceMetadata(String name) + @Override + public Map getDataSourceInformationMap() { - return tables.get(name); + throw new UnsupportedOperationException(); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java index a51aea811e93..fc6d94da2eb7 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfig.java @@ -23,14 +23,25 @@ import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.joda.time.Period; +/** + * Broker-side configuration class for managing segment polling from the Coordinator and + * customizing properties related to the SegmentMetadata cache. + *

    + * The property {@link #awaitInitializationOnStart} is overridden in this class with a default value + * of {@code true}, which differs from the parent class. This ensures that the SegmentMetadata cache is + * fully initialized before other startup processes proceed. + */ public class BrokerSegmentMetadataCacheConfig extends SegmentMetadataCacheConfig { + // A flag indicating whether to cache polled segments from the Coordinator. @JsonProperty private boolean metadataSegmentCacheEnable = true; + // Interval for polling segments from the coordinator. @JsonProperty private long metadataSegmentPollPeriod = 60000; + // A flag indicating whether to wait for cache initialization during startup. @JsonProperty private boolean awaitInitializationOnStart = true; @@ -58,6 +69,9 @@ public long getMetadataSegmentPollPeriod() return metadataSegmentPollPeriod; } + /** + * This property is overriden on the broker, so that the cache initialization blocks startup. + */ @Override public boolean isAwaitInitializationOnStart() { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java index 4f8044c48d45..50a506d0e6f3 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java @@ -53,7 +53,7 @@ /** * This class polls the Coordinator in background to keep the latest segments. - * Provides {@link #getSegments()} for others to get segments in metadata store. + * Provides {@link #getSegments()} for others to get the segments. * * The difference between this class and {@link SegmentsMetadataManager} is that this class resides * in Broker's memory, while {@link SegmentsMetadataManager} resides in Coordinator's memory. In diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index 5b344621f1f8..abd6f6af4d43 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -22,12 +22,15 @@ import com.google.inject.Inject; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.TableDataSource; -import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.SegmentManager; -import org.apache.druid.sql.calcite.table.DatasourceTable; +import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; +/** + * Builds {@link PhysicalDatasourceMetadata} for a dataSource, including information about its schema, + * joinability, and broadcast status. + */ public class PhysicalDatasourceMetadataBuilder { private final JoinableFactory joinableFactory; @@ -40,10 +43,12 @@ public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, Segmen this.segmentManager = segmentManager; } - DatasourceTable.PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) + /** + * Builds physical metadata for the given data source. + */ + PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) { final String dataSource = dataSourceInformation.getDatasource(); - final RowSignature rowSignature = dataSourceInformation.getRowSignature(); final TableDataSource tableDataSource; // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing @@ -59,6 +64,11 @@ DatasourceTable.PhysicalDatasourceMetadata build(DataSourceInformation dataSourc } else { tableDataSource = new TableDataSource(dataSource); } - return new DatasourceTable.PhysicalDatasourceMetadata(tableDataSource, rowSignature, isJoinable, isBroadcast); + return new PhysicalDatasourceMetadata( + tableDataSource, + dataSourceInformation.getRowSignature(), + isJoinable, + isBroadcast + ); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index 5480cd61f352..33c76b0d2265 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -83,6 +83,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -234,6 +235,9 @@ public void refresh( return runningSchema; } + /** + * Test the case when coordinator returns information for all the requested dataSources + */ @Test public void testGetAllDsSchemaFromCoordinator() throws InterruptedException { @@ -280,6 +284,10 @@ public ListenableFuture> fetchDataSourceInformation( Assert.assertEquals(foo3RowSignature, schema.getPhysicalDatasourceMetadata("foo3").rowSignature()); } + /** + * Test the case when Coordinator returns information for a subset of dataSources. + * Check if SegmentMetadataQuery is fired for segments of the remaining dataSources. + */ @Test public void testGetFewDsSchemaFromCoordinator() throws InterruptedException { @@ -700,7 +708,12 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception public void testStaleDatasourceRefresh() throws IOException, InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - checkStaleDatasourceRefresh(schema); + Set segments = new HashSet<>(); + Set datasources = new HashSet<>(); + datasources.add("wat"); + Assert.assertNull(schema.getPhysicalDatasourceMetadata("wat")); + schema.refresh(segments, datasources); + Assert.assertNull(schema.getPhysicalDatasourceMetadata("wat")); } @Test From b9fb83dbd54345e191aa9b2c563e0567d70e731d Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 13 Sep 2023 15:25:28 +0530 Subject: [PATCH 31/36] Minor refactoring in SegmentMetadataCache --- .../metadata/SegmentMetadataCache.java | 41 ++++++++----------- .../schema/BrokerSegmentMetadataCache.java | 35 ++++++---------- 2 files changed, 31 insertions(+), 45 deletions(-) diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 376d4582d5d4..fd59ff397b9d 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -93,7 +93,7 @@ /** * Coordinator-side cache of segment metadata that combines segments to identify - * datasources.The cache provides metadata about a dataSource, see {@link DataSourceInformation}. + * datasources. The cache provides metadata about a dataSource, see {@link DataSourceInformation}. */ @ManageLifecycle public class SegmentMetadataCache @@ -411,23 +411,18 @@ public void refresh(final Set segmentsToRefresh, final Set da // Rebuild the dataSources. for (String dataSource : dataSourcesToRebuild) { - rebuildDatasource(dataSource); - } - } - - protected void rebuildDatasource(String dataSource) - { - final DataSourceInformation druidTable = buildDruidTable(dataSource); - if (druidTable == null) { - log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tableManager.removeFromTable(dataSource); - return; - } - final DataSourceInformation oldTable = tableManager.put(dataSource, druidTable); - if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { - log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); - } else { - log.debug("[%s] signature is unchanged.", dataSource); + final DataSourceInformation druidTable = buildDruidTable(dataSource); + if (druidTable == null) { + log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); + tableManager.removeFromTable(dataSource); + return; + } + final DataSourceInformation oldTable = tableManager.put(dataSource, druidTable); + if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { + log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); + } else { + log.debug("[%s] signature is unchanged.", dataSource); + } } } @@ -716,7 +711,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin throw new ISE("'segments' must all match 'dataSource'!"); } - log.info("Refreshing metadata for dataSource[%s].", dataSource); + log.debug("Refreshing metadata for dataSource[%s].", dataSource); final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, dataSource); @@ -742,7 +737,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin log.warn("Got analysis for segment [%s] we didn't ask for, ignoring.", analysis.getId()); } else { final RowSignature rowSignature = analysisToRowSignature(analysis); - log.info("Segment[%s] has signature[%s].", segmentId, rowSignature); + log.debug("Segment[%s] has signature[%s].", segmentId, rowSignature); segmentMetadataInfo.compute( dataSource, (datasourceKey, dataSourceSegments) -> { @@ -794,7 +789,7 @@ private Set refreshSegmentsForDataSource(final String dataSource, fin emitter.emit(builder.setMetric("metadatacache/refresh/time", refreshDurationMillis)); - log.info( + log.debug( "Refreshed metadata for dataSource [%s] in %,d ms (%d segments queried, %d segments left).", dataSource, refreshDurationMillis, @@ -934,7 +929,7 @@ public Sequence runSegmentMetadataQuery( } @VisibleForTesting - public static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) + static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) { final RowSignature.Builder rowSignatureBuilder = RowSignature.builder(); for (Map.Entry entry : analysis.getColumns().entrySet()) { @@ -971,7 +966,7 @@ public static RowSignature analysisToRowSignature(final SegmentAnalysis analysis * This method is not thread-safe and must be used only in unit tests. */ @VisibleForTesting - public void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) + void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) { final ConcurrentSkipListMap dataSourceSegments = segmentMetadataInfo .computeIfAbsent( diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 90fdec9c54eb..e23a396be7db 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -147,28 +147,19 @@ public void refresh(final Set segmentsToRefresh, final Set da // Rebuild the dataSources. for (String dataSource : dataSourcesToRebuild) { - rebuildDatasource(dataSource); - } - } - - /** - * Build table schema and convert DataSourceInformation to PhysicalDatasourceMetadata. - */ - @Override - protected void rebuildDatasource(String dataSource) - { - final DataSourceInformation druidTable = buildDruidTable(dataSource); - if (druidTable == null) { - log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tableManager.removeFromTable(dataSource); - return; - } - final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); - final PhysicalDatasourceMetadata oldTable = tableManager.put(dataSource, physicalDatasourceMetadata); - if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { - log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); - } else { - log.debug("[%s] signature is unchanged.", dataSource); + final DataSourceInformation druidTable = buildDruidTable(dataSource); + if (druidTable == null) { + log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); + tableManager.removeFromTable(dataSource); + return; + } + final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); + final PhysicalDatasourceMetadata oldTable = tableManager.put(dataSource, physicalDatasourceMetadata); + if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { + log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); + } else { + log.debug("[%s] signature is unchanged.", dataSource); + } } } From aa2bfe790b620dab635fad38de8398803284f7de Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Wed, 13 Sep 2023 19:36:54 +0530 Subject: [PATCH 32/36] Make SegmentMetadataCache generic --- ...ruidSchemaInternRowSignatureBenchmark.java | 3 +- .../metadata/DataSourceInformation.java | 19 ++--- .../metadata/SegmentMetadataCache.java | 71 ++++--------------- .../druid/server/http/MetadataResource.java | 4 +- .../metadata/SegmentMetadataCacheTest.java | 50 ++++++------- .../org/apache/druid/cli/CliCoordinator.java | 48 ++++++++++++- .../schema/BrokerSegmentMetadataCache.java | 43 +++-------- .../druid/sql/calcite/schema/DruidSchema.java | 2 +- .../PhysicalDatasourceMetadataBuilder.java | 2 +- .../sql/calcite/table/DatasourceTable.java | 37 +++++----- .../BrokerSegmentMetadataCacheTest.java | 32 ++++----- ...PhysicalDataSourceMetadataBuilderTest.java | 8 +-- 12 files changed, 146 insertions(+), 173 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 5f0634f283a4..c6a0360fea39 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -30,6 +30,7 @@ import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; @@ -68,7 +69,7 @@ public class DruidSchemaInternRowSignatureBenchmark { private SegmentMetadataCacheForBenchmark cache; - private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCache + private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCache { public SegmentMetadataCacheForBenchmark( final QueryLifecycleFactory queryLifecycleFactory, diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index 1fe6534d923c..f25c104203c5 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; import org.apache.druid.segment.column.RowSignature; import java.util.Objects; @@ -30,22 +31,22 @@ */ public class DataSourceInformation { - private final String datasource; + private final String dataSource; private final RowSignature rowSignature; @JsonCreator public DataSourceInformation( - @JsonProperty("datasource") String datasource, + @JsonProperty("datasource") String dataSource, @JsonProperty("rowSignature") RowSignature rowSignature) { - this.datasource = datasource; - this.rowSignature = rowSignature; + this.dataSource = Preconditions.checkNotNull(dataSource, "'dataSource' must be nonnull"); + this.rowSignature = Preconditions.checkNotNull(rowSignature, "rowSignature"); } @JsonProperty - public String getDatasource() + public String getDataSource() { - return datasource; + return dataSource; } @JsonProperty @@ -64,7 +65,7 @@ public boolean equals(Object o) return false; } DataSourceInformation that = (DataSourceInformation) o; - return Objects.equals(datasource, that.datasource) && Objects.equals( + return Objects.equals(dataSource, that.dataSource) && Objects.equals( rowSignature, that.rowSignature ); @@ -73,14 +74,14 @@ public boolean equals(Object o) @Override public int hashCode() { - return Objects.hash(datasource, rowSignature); + return Objects.hash(dataSource, rowSignature); } @Override public String toString() { return "DataSourceSchema{" + - "datasource='" + datasource + '\'' + + "dataSource='" + dataSource + '\'' + ", rowSignature=" + rowSignature + '}'; } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index fd59ff397b9d..2ff4640f1297 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -96,7 +96,7 @@ * datasources. The cache provides metadata about a dataSource, see {@link DataSourceInformation}. */ @ManageLifecycle -public class SegmentMetadataCache +public class SegmentMetadataCache { // Newest segments first, so they override older ones. private static final Comparator SEGMENT_ORDER = Comparator @@ -119,11 +119,10 @@ public class SegmentMetadataCache private final ColumnTypeMergePolicy columnTypeMergePolicy; /** - * Manages tables of DataSourceInformation. This manager is used to retrieve and store - * information related to dataSources. + * Map of dataSource -> DataSourceInformation * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final TableManager tableManager = new TableManager<>(); + protected final ConcurrentMap tables = new ConcurrentHashMap<>(); /** * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. @@ -378,6 +377,7 @@ private void startCacheExec() @LifecycleStart public void start() throws InterruptedException { + log.info("Starting SegmentMetadataCache."); startCacheExec(); if (config.isAwaitInitializationOnStart()) { @@ -414,10 +414,10 @@ public void refresh(final Set segmentsToRefresh, final Set da final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tableManager.removeFromTable(dataSource); + tables.remove(dataSource); return; } - final DataSourceInformation oldTable = tableManager.put(dataSource, druidTable); + final DataSourceInformation oldTable = tables.put(dataSource, (T) druidTable); if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { @@ -438,19 +438,19 @@ public void awaitInitialization() throws InterruptedException initialized.await(); } - public DataSourceInformation getDatasource(String name) + public T getDatasource(String name) { - return tableManager.get(name); + return tables.get(name); } - public Map getDataSourceInformationMap() + public Map getDataSourceInformationMap() { - return tableManager.getAll(); + return ImmutableMap.copyOf(tables); } public Set getDatasourceNames() { - return tableManager.getKeySet(); + return tables.keySet(); } @VisibleForTesting @@ -517,7 +517,7 @@ public void addSegment(final DruidServerMetadata server, final DataSegment segme } ); } - if (!tableManager.tablesContains(segment.getDataSource())) { + if (!tables.containsKey(segment.getDataSource())) { refreshImmediately = true; } @@ -548,7 +548,7 @@ public void removeSegment(final DataSegment segment) totalSegments--; } if (segmentsMap.isEmpty()) { - tableManager.removeFromTable(segment.getDataSource()); + tables.remove(segment.getDataSource()); log.info("dataSource [%s] no longer exists, all metadata removed.", segment.getDataSource()); return null; } else { @@ -1111,49 +1111,4 @@ public String toString() return NAME; } } - - /** - * A generic class for managing table. - * - * @param The type of data associated with the tables (e.g., DataSourceInformation, PhysicalDataSourceMetadata). - */ - public static class TableManager - { - private final ConcurrentMap tables = new ConcurrentHashMap<>(); - - public void removeFromTable(String s) - { - tables.remove(s); - } - - public boolean tablesContains(String s) - { - return tables.containsKey(s); - } - - public T get(String s) - { - return tables.get(s); - } - - public Set getKeySet() - { - return tables.keySet(); - } - - public Map getAll() - { - return ImmutableMap.copyOf(tables); - } - - public T put(String key, T value) - { - return tables.put(key, value); - } - - public void putAll(Map anotherTable) - { - tables.putAll(anotherTable); - } - } } diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index c79962c0d111..7fe2cd76e129 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -75,7 +75,7 @@ public class MetadataResource private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; private final DruidCoordinator coordinator; - private final @Nullable SegmentMetadataCache segmentMetadataCache; + private final @Nullable SegmentMetadataCache segmentMetadataCache; @Inject public MetadataResource( @@ -83,7 +83,7 @@ public MetadataResource( IndexerMetadataStorageCoordinator metadataStorageCoordinator, AuthorizerMapper authorizerMapper, DruidCoordinator coordinator, - @Nullable SegmentMetadataCache segmentMetadataCache + @Nullable SegmentMetadataCache segmentMetadataCache ) { this.segmentsMetadataManager = segmentsMetadataManager; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index cd664f13504e..78ba2add3627 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -63,7 +63,7 @@ public class SegmentMetadataCacheTest extends SegmentMetadataCacheCommon // Timeout to allow (rapid) debugging, while not blocking tests with errors. private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); - private SegmentMetadataCache runningSchema; + private SegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); @@ -85,15 +85,15 @@ public void tearDown() throws Exception walker.close(); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException + public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException + public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException { Preconditions.checkState(runningSchema == null); - runningSchema = new SegmentMetadataCache( + runningSchema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, config, @@ -126,7 +126,7 @@ public void markDataSourceAsNeedRebuild(String datasource) @Test public void testGetTableMap() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); final Set tableNames = schema.getDatasourceNames(); @@ -136,7 +136,7 @@ public void testGetTableMap() throws InterruptedException @Test public void testGetTableMapFoo() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -164,7 +164,7 @@ public void testGetTableMapFoo() throws InterruptedException @Test public void testGetTableMapFoo2() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo2"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -185,7 +185,7 @@ public void testGetTableMapSomeTable() throws InterruptedException { // using 'newest first' column type merge strategy, the types are expected to be the types defined in the newer // segment, except for json, which is special handled - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch( + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch( new SegmentMetadataCacheConfig() { @Override public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() @@ -232,7 +232,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt { // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); final RowSignature fooRowSignature = fooDs.getRowSignature(); @@ -276,28 +276,28 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt @Test public void testAvailableSegmentMetadataNumRows() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkAvailableSegmentMetadataNumRows(schema); } @Test public void testNullDatasource() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullDatasource(schema); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullAvailableSegmentMetadata(schema); } @Test public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); final List segments = segmentsMetadata.values() .stream() @@ -354,7 +354,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -395,7 +395,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -440,7 +440,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -482,7 +482,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -521,7 +521,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -577,7 +577,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -636,7 +636,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr { String datasource = "serverSegmentRemoveTest"; CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -669,7 +669,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -715,7 +715,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -785,7 +785,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); // Need to create schema for this test because the available schemas don't mock the QueryLifecycleFactory, which I need for this test. - SegmentMetadataCache mySchema = new SegmentMetadataCache( + SegmentMetadataCache mySchema = new SegmentMetadataCache( factoryMock, serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -905,7 +905,7 @@ public void testSegmentMetadataFallbackType() @Test public void testStaleDatasourceRefresh() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Set segments = new HashSet<>(); Set datasources = new HashSet<>(); datasources.add("wat"); @@ -920,7 +920,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept String dataSource = "xyz"; CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 6dbd99d3c0d3..b111c1147cc8 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -74,6 +74,7 @@ import org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory; import org.apache.druid.java.util.common.lifecycle.Lifecycle; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.metadata.MetadataRuleManager; import org.apache.druid.metadata.MetadataRuleManagerConfig; @@ -87,9 +88,11 @@ import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.lookup.LookupSerdeModule; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; +import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.ClientQuerySegmentWalker; +import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.audit.AuditManagerProvider; import org.apache.druid.server.coordinator.CoordinatorConfigManager; import org.apache.druid.server.coordinator.DruidCoordinator; @@ -125,6 +128,7 @@ import org.apache.druid.server.lookup.cache.LookupCoordinatorManagerConfig; import org.apache.druid.server.metrics.ServiceStatusMonitor; import org.apache.druid.server.router.TieredBrokerConfig; +import org.apache.druid.server.security.Escalator; import org.eclipse.jetty.server.Server; import org.joda.time.Duration; @@ -363,7 +367,6 @@ private boolean isSegmentMetadataCacheEnabled() return Boolean.parseBoolean(properties.getProperty(SEGMENT_METADATA_CACHE_ENABLED)); } - private static class CoordinatorCustomDutyGroupsProvider implements Provider { private Properties props; @@ -481,7 +484,48 @@ public void configure(Binder binder) binder.bind(CoordinatorTimeline.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); binder.bind(TimelineServerView.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, QueryableCoordinatorServerView.class); - LifecycleModule.register(binder, SegmentMetadataCache.class); + binder.bind(new TypeLiteral>() {}) + .toProvider(SegmentMetadataCacheProvider.class).in(ManageLifecycle.class); + } + + private static class SegmentMetadataCacheProvider implements Provider> + { + private final QueryLifecycleFactory queryLifecycleFactory; + private final TimelineServerView serverView; + private final SegmentMetadataCacheConfig config; + private final Escalator escalator; + private final InternalQueryConfig internalQueryConfig; + private final ServiceEmitter emitter; + + @Inject + public SegmentMetadataCacheProvider( + QueryLifecycleFactory queryLifecycleFactory, + TimelineServerView serverView, + SegmentMetadataCacheConfig config, + Escalator escalator, + InternalQueryConfig internalQueryConfig, + ServiceEmitter emitter + ) + { + this.queryLifecycleFactory = queryLifecycleFactory; + this.serverView = serverView; + this.config = config; + this.escalator = escalator; + this.internalQueryConfig = internalQueryConfig; + this.emitter = emitter; + } + + @Override + public SegmentMetadataCache get() + { + return new SegmentMetadataCache<>( + queryLifecycleFactory, + serverView, + config, + escalator, + internalQueryConfig, + emitter); + } } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index e23a396be7db..14803ea31417 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -44,9 +44,9 @@ /** * Broker-side cache of segment metadata that combines segments to identify - * datasources which become "tables" in Calcite. This cache provides the "physical" - * metadata about a datasource which is blended with catalog "logical" metadata - * to provide the final user-view of each datasource. + * dataSources which become "tables" in Calcite. This cache provides the "physical" + * metadata about a dataSource which is blended with catalog "logical" metadata + * to provide the final user-view of each dataSource. *

    * This class extends {@link SegmentMetadataCache} and introduces following changes, *

      @@ -57,7 +57,7 @@ *
    */ @ManageLifecycle -public class BrokerSegmentMetadataCache extends SegmentMetadataCache +public class BrokerSegmentMetadataCache extends SegmentMetadataCache { private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataCache.class); @@ -68,7 +68,7 @@ public class BrokerSegmentMetadataCache extends SegmentMetadataCache * information related to dataSources. * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. */ - private final TableManager tableManager = new TableManager<>(); + //private final TableManager<> tableManager = new TableManager<>(); private final CoordinatorClient coordinatorClient; @Inject @@ -113,7 +113,7 @@ public void refresh(final Set segmentsToRefresh, final Set da try { FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) .forEach(item -> polledDataSourceMetadata.put( - item.getDatasource(), + item.getDataSource(), physicalDatasourceMetadataBuilder.build(item) )); } @@ -124,7 +124,7 @@ public void refresh(final Set segmentsToRefresh, final Set da // remove any extra dataSources returned polledDataSourceMetadata.keySet().removeIf(Predicates.not(dataSourcesToQuery::contains)); - tableManager.putAll(polledDataSourceMetadata); + tables.putAll(polledDataSourceMetadata); // Remove segments of the dataSource from refresh list for which we received schema from the Coordinator. segmentsToRefresh.removeIf(segmentId -> polledDataSourceMetadata.containsKey(segmentId.getDataSource())); @@ -150,39 +150,16 @@ public void refresh(final Set segmentsToRefresh, final Set da final DataSourceInformation druidTable = buildDruidTable(dataSource); if (druidTable == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); - tableManager.removeFromTable(dataSource); + tables.remove(dataSource); return; } final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); - final PhysicalDatasourceMetadata oldTable = tableManager.put(dataSource, physicalDatasourceMetadata); - if (oldTable == null || !oldTable.rowSignature().equals(physicalDatasourceMetadata.rowSignature())) { + final PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); + if (oldTable == null || !oldTable.getRowSignature().equals(physicalDatasourceMetadata.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { log.debug("[%s] signature is unchanged.", dataSource); } } } - - @Override - public Set getDatasourceNames() - { - return tableManager.getKeySet(); - } - - public PhysicalDatasourceMetadata getPhysicalDatasourceMetadata(String name) - { - return tableManager.get(name); - } - - @Override - public DataSourceInformation getDatasource(String name) - { - throw new UnsupportedOperationException(); - } - - @Override - public Map getDataSourceInformationMap() - { - throw new UnsupportedOperationException(); - } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java index 69b96403aa82..580b9acbb520 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/DruidSchema.java @@ -55,7 +55,7 @@ public Table getTable(String name) if (druidSchemaManager != null) { return druidSchemaManager.getTable(name); } else { - DatasourceTable.PhysicalDatasourceMetadata dsMetadata = segmentMetadataCache.getPhysicalDatasourceMetadata(name); + DatasourceTable.PhysicalDatasourceMetadata dsMetadata = segmentMetadataCache.getDatasource(name); return dsMetadata == null ? null : new DatasourceTable(dsMetadata); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index abd6f6af4d43..c5922b066885 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -48,7 +48,7 @@ public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, Segmen */ PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) { - final String dataSource = dataSourceInformation.getDatasource(); + final String dataSource = dataSourceInformation.getDataSource(); final TableDataSource tableDataSource; // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/table/DatasourceTable.java b/sql/src/main/java/org/apache/druid/sql/calcite/table/DatasourceTable.java index 8b6903133a95..eeac85a2c1ed 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/table/DatasourceTable.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/table/DatasourceTable.java @@ -26,6 +26,7 @@ import org.apache.druid.query.DataSource; import org.apache.druid.query.TableDataSource; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.metadata.DataSourceInformation; import java.util.Objects; @@ -44,34 +45,28 @@ public class DatasourceTable extends DruidTable * published in the Coordinator. Used only for datasources, since only * datasources are computed from segments. */ - public static class PhysicalDatasourceMetadata + public static class PhysicalDatasourceMetadata extends DataSourceInformation { - private final TableDataSource dataSource; - private final RowSignature rowSignature; + private final TableDataSource tableDataSource; private final boolean joinable; private final boolean broadcast; public PhysicalDatasourceMetadata( - final TableDataSource dataSource, + final TableDataSource tableDataSource, final RowSignature rowSignature, final boolean isJoinable, final boolean isBroadcast ) { - this.dataSource = Preconditions.checkNotNull(dataSource, "dataSource"); - this.rowSignature = Preconditions.checkNotNull(rowSignature, "rowSignature"); + super(tableDataSource.getName(), rowSignature); + this.tableDataSource = Preconditions.checkNotNull(tableDataSource, "dataSource"); this.joinable = isJoinable; this.broadcast = isBroadcast; } public TableDataSource dataSource() { - return dataSource; - } - - public RowSignature rowSignature() - { - return rowSignature; + return tableDataSource; } public boolean isJoinable() @@ -93,20 +88,20 @@ public boolean equals(Object o) if (o == null || getClass() != o.getClass()) { return false; } + if (!super.equals(o)) { + return false; + } PhysicalDatasourceMetadata that = (PhysicalDatasourceMetadata) o; - if (!Objects.equals(dataSource, that.dataSource)) { - return false; - } - return Objects.equals(rowSignature, that.rowSignature); + return Objects.equals(tableDataSource, that.tableDataSource); } @Override public int hashCode() { - int result = dataSource != null ? dataSource.hashCode() : 0; - result = 31 * result + (rowSignature != null ? rowSignature.hashCode() : 0); + int result = tableDataSource != null ? tableDataSource.hashCode() : 0; + result = 31 * result + super.hashCode(); return result; } @@ -114,8 +109,8 @@ public int hashCode() public String toString() { return "DatasourceMetadata{" + - "dataSource=" + dataSource + - ", rowSignature=" + rowSignature + + "dataSource=" + tableDataSource + + ", rowSignature=" + getRowSignature() + '}'; } } @@ -126,7 +121,7 @@ public DatasourceTable( final PhysicalDatasourceMetadata physicalMetadata ) { - super(physicalMetadata.rowSignature()); + super(physicalMetadata.getRowSignature()); this.physicalMetadata = physicalMetadata; } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index 33c76b0d2265..f4b55fb5929a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -278,10 +278,10 @@ public ListenableFuture> fetchDataSourceInformation( final Set tableNames = schema.getDatasourceNames(); Assert.assertEquals(ImmutableSet.of(CalciteTests.DATASOURCE1, CalciteTests.DATASOURCE2, CalciteTests.SOME_DATASOURCE, "foo3"), tableNames); - Assert.assertEquals(dataSource1RowSignature, schema.getPhysicalDatasourceMetadata(DATASOURCE1).rowSignature()); - Assert.assertEquals(dataSource2RowSignature, schema.getPhysicalDatasourceMetadata(DATASOURCE2).rowSignature()); - Assert.assertEquals(someDataSourceRowSignature, schema.getPhysicalDatasourceMetadata(SOME_DATASOURCE).rowSignature()); - Assert.assertEquals(foo3RowSignature, schema.getPhysicalDatasourceMetadata("foo3").rowSignature()); + Assert.assertEquals(dataSource1RowSignature, schema.getDatasource(DATASOURCE1).getRowSignature()); + Assert.assertEquals(dataSource2RowSignature, schema.getDatasource(DATASOURCE2).getRowSignature()); + Assert.assertEquals(someDataSourceRowSignature, schema.getDatasource(SOME_DATASOURCE).getRowSignature()); + Assert.assertEquals(foo3RowSignature, schema.getDatasource("foo3").getRowSignature()); } /** @@ -355,7 +355,7 @@ public void testGetTableMap() throws InterruptedException public void testGetTableMapFoo() throws InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata("foo"); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource("foo"); final DruidTable fooTable = new DatasourceTable(fooDs); final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); final List fields = rowType.getFieldList(); @@ -385,7 +385,7 @@ public void testGetTableMapFoo() throws InterruptedException public void testGetTableMapFoo2() throws InterruptedException { BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata("foo2"); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource("foo2"); final DruidTable fooTable = new DatasourceTable(fooDs); final RelDataType rowType = fooTable.getRowType(new JavaTypeFactoryImpl()); final List fields = rowType.getFieldList(); @@ -417,7 +417,7 @@ public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePoli }, new NoopCoordinatorClient() ); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata(CalciteTests.SOME_DATASOURCE); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource(CalciteTests.SOME_DATASOURCE); final DruidTable table = new DatasourceTable(fooDs); final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); final List fields = rowType.getFieldList(); @@ -460,7 +460,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); - final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getPhysicalDatasourceMetadata(CalciteTests.SOME_DATASOURCE); + final DatasourceTable.PhysicalDatasourceMetadata fooDs = schema.getDatasource(CalciteTests.SOME_DATASOURCE); final DruidTable table = new DatasourceTable(fooDs); final RelDataType rowType = table.getRowType(new JavaTypeFactoryImpl()); final List fields = rowType.getFieldList(); @@ -541,7 +541,7 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte { BrokerSegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getPhysicalDatasourceMetadata("foo"); + DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); @@ -573,7 +573,7 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); Assert.assertTrue(fooTable.dataSource() instanceof GlobalTableDataSource); @@ -594,7 +594,7 @@ public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws Inte refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); @@ -607,7 +607,7 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw { BrokerSegmentMetadataCache schema = buildSchemaMarkAndRefreshLatch(); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getPhysicalDatasourceMetadata("foo"); + DatasourceTable.PhysicalDatasourceMetadata fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); @@ -640,7 +640,7 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); // Should not be a GlobalTableDataSource for now, because isGlobal is couple with joinability. Ideally this will be @@ -662,7 +662,7 @@ public void testLocalSegmentCacheSetsDataSourceAsBroadcastButNotJoinable() throw refreshLatch = new CountDownLatch(1); Assert.assertTrue(refreshLatch.await(WAIT_TIMEOUT_SECS, TimeUnit.SECONDS)); - fooTable = schema.getPhysicalDatasourceMetadata("foo"); + fooTable = schema.getDatasource("foo"); Assert.assertNotNull(fooTable); Assert.assertTrue(fooTable.dataSource() instanceof TableDataSource); Assert.assertFalse(fooTable.dataSource() instanceof GlobalTableDataSource); @@ -711,9 +711,9 @@ public void testStaleDatasourceRefresh() throws IOException, InterruptedExceptio Set segments = new HashSet<>(); Set datasources = new HashSet<>(); datasources.add("wat"); - Assert.assertNull(schema.getPhysicalDatasourceMetadata("wat")); + Assert.assertNull(schema.getDatasource("wat")); schema.refresh(segments, datasources); - Assert.assertNull(schema.getPhysicalDatasourceMetadata("wat")); + Assert.assertNull(schema.getDatasource("wat")); } @Test diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java index 987a6d1e0100..ccdbef25461b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java @@ -115,13 +115,13 @@ public void testBuild() DatasourceTable.PhysicalDatasourceMetadata fooDs = physicalDatasourceMetadataBuilder.build(foo); Assert.assertTrue(fooDs.isJoinable()); Assert.assertTrue(fooDs.isBroadcast()); - Assert.assertEquals(fooDs.dataSource().getName(), foo.getDatasource()); - Assert.assertEquals(fooDs.rowSignature(), foo.getRowSignature()); + Assert.assertEquals(fooDs.dataSource().getName(), foo.getDataSource()); + Assert.assertEquals(fooDs.getRowSignature(), foo.getRowSignature()); DatasourceTable.PhysicalDatasourceMetadata barDs = physicalDatasourceMetadataBuilder.build(bar); Assert.assertFalse(barDs.isJoinable()); Assert.assertFalse(barDs.isBroadcast()); - Assert.assertEquals(barDs.dataSource().getName(), bar.getDatasource()); - Assert.assertEquals(barDs.rowSignature(), bar.getRowSignature()); + Assert.assertEquals(barDs.dataSource().getName(), bar.getDataSource()); + Assert.assertEquals(barDs.getRowSignature(), bar.getRowSignature()); } } From e97dcda911a9d32fc61853b6c5a46743369f9eee Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 00:04:21 +0530 Subject: [PATCH 33/36] Add a generic abstract class for segment metadata cache --- ...ruidSchemaInternRowSignatureBenchmark.java | 3 +- .../QueryableCoordinatorServerView.java | 13 +- .../AbstractSegmentMetadataCache.java | 1084 +++++++++++++++++ .../metadata/SegmentMetadataCache.java | 1054 +--------------- .../metadata/SegmentMetadataCacheConfig.java | 6 +- .../druid/server/http/MetadataResource.java | 4 +- .../SegmentDataCacheConcurrencyTest.java | 19 +- .../metadata/SegmentMetadataCacheCommon.java | 10 +- .../SegmentMetadataCacheConfigTest.java | 4 +- .../metadata/SegmentMetadataCacheTest.java | 64 +- .../org/apache/druid/cli/CliCoordinator.java | 50 +- .../schema/BrokerSegmentMetadataCache.java | 43 +- .../PhysicalDatasourceMetadataBuilder.java | 6 +- .../BrokerSegmentMetadataCacheConfigTest.java | 6 +- .../BrokerSegmentMetadataCacheTest.java | 14 +- ...PhysicalDataSourceMetadataBuilderTest.java | 43 +- 16 files changed, 1216 insertions(+), 1207 deletions(-) create mode 100644 server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index c6a0360fea39..5f0634f283a4 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -30,7 +30,6 @@ import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; @@ -69,7 +68,7 @@ public class DruidSchemaInternRowSignatureBenchmark { private SegmentMetadataCacheForBenchmark cache; - private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCache + private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCache { public SegmentMetadataCacheForBenchmark( final QueryLifecycleFactory queryLifecycleFactory, diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index 0f69f2699a7f..1db03afa2f85 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -32,6 +32,7 @@ import org.apache.druid.query.DataSource; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; +import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.VersionedIntervalTimeline; @@ -43,13 +44,13 @@ /** * ServerView of coordinator for the state of segments being loaded in the cluster. - *
    - * This class extends {@link BrokerServerView} and implements {@link CoordinatorTimeline}. + * + *

    This class extends {@link BrokerServerView} and implements {@link CoordinatorTimeline}. * The main distinction between this class and {@link CoordinatorServerView} is the maintenance of a timeline - * of {@link ServerSelector} objects, while the other class stores {@link SegmentLoadInfo} object in its timeline. - *
    - * A new timeline class (implementing {@link TimelineServerView}) is required for - * {@link org.apache.druid.segment.metadata.SegmentMetadataCache}, which will run on the Coordinator. + * of {@link ServerSelector} objects, while the other class stores {@link SegmentLoadInfo} object in its timeline.

    + * + *

    A new timeline class (implementing {@link TimelineServerView}) is required for + * {@link AbstractSegmentMetadataCache}, which will run on the Coordinator.

    */ @ManageLifecycle public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline diff --git a/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java new file mode 100644 index 000000000000..aefc16b679b3 --- /dev/null +++ b/server/src/main/java/org/apache/druid/segment/metadata/AbstractSegmentMetadataCache.java @@ -0,0 +1,1084 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.metadata; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicates; +import com.google.common.base.Stopwatch; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Interner; +import com.google.common.collect.Interners; +import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; +import com.google.errorprone.annotations.concurrent.GuardedBy; +import org.apache.druid.client.InternalQueryConfig; +import org.apache.druid.client.ServerView; +import org.apache.druid.client.TimelineServerView; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Yielder; +import org.apache.druid.java.util.common.guava.Yielders; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.lifecycle.LifecycleStop; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; +import org.apache.druid.query.DruidMetrics; +import org.apache.druid.query.QueryContexts; +import org.apache.druid.query.TableDataSource; +import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; +import org.apache.druid.query.metadata.metadata.ColumnAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentAnalysis; +import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; +import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.Types; +import org.apache.druid.server.QueryLifecycleFactory; +import org.apache.druid.server.coordination.DruidServerMetadata; +import org.apache.druid.server.coordination.ServerType; +import org.apache.druid.server.security.Access; +import org.apache.druid.server.security.Escalator; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.SegmentId; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * An abstract class that listens for segment change events and caches segment metadata. + * It periodically queries data nodes to fetch segment schemas and combines them to build a data source schema. + * + *

    This class is generic and is parameterized by a type {@code T} that extends {@link DataSourceInformation}.

    + * + *

    This class has an abstract method {@link #refresh(Set, Set)} which the child class must override + * with the logic to get segment schema.

    + * + * @param The type of information associated with the data source, which must extend {@link DataSourceInformation}. + */ +public abstract class AbstractSegmentMetadataCache +{ + // Newest segments first, so they override older ones. + private static final Comparator SEGMENT_ORDER = Comparator + .comparing((SegmentId segmentId) -> segmentId.getInterval().getStart()) + .reversed() + .thenComparing(Function.identity()); + + private static final EmittingLogger log = new EmittingLogger(AbstractSegmentMetadataCache.class); + private static final int MAX_SEGMENTS_PER_QUERY = 15000; + private static final long DEFAULT_NUM_ROWS = 0; + private static final Interner ROW_SIGNATURE_INTERNER = Interners.newWeakInterner(); + private final QueryLifecycleFactory queryLifecycleFactory; + private final SegmentMetadataCacheConfig config; + // Escalator, so we can attach an authentication result to queries we generate. + private final Escalator escalator; + + private final ExecutorService cacheExec; + private final ExecutorService callbackExec; + private final ServiceEmitter emitter; + private final ColumnTypeMergePolicy columnTypeMergePolicy; + + /** + * Map of dataSource and generic object extending DataSourceInformation. + * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. + */ + protected final ConcurrentMap tables = new ConcurrentHashMap<>(); + + /** + * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. + * Use SortedMap for segments so they are merged in deterministic order, from older to newer. + * + * This map is updated by these two threads. + * + * - {@link #callbackExec} can update it in {@link #addSegment}, {@link #removeServerSegment}, + * and {@link #removeSegment}. + * - {@link #cacheExec} can update it in {@link #refreshSegmentsForDataSource}. + * + * While it is being updated, this map is read by these two types of thread. + * + * - {@link #cacheExec} can iterate all {@link AvailableSegmentMetadata}s per datasource. + * See {@link #buildDruidTable}. + * - Query threads can create a snapshot of the entire map for processing queries on the system table. + * See {@link #getSegmentMetadataSnapshot()}. + * + * As the access pattern of this map is read-intensive, we should minimize the contention between writers and readers. + * Since there are two threads that can update this map at the same time, those writers should lock the inner map + * first and then lock the entry before it updates segment metadata. This can be done using + * {@link ConcurrentMap#compute} as below. Note that, if you need to update the variables guarded by {@link #lock} + * inside of compute(), you should get the lock before calling compute() to keep the function executed in compute() + * not expensive. + * + * 
    +   *   segmentMedataInfo.compute(
+   *     datasourceParam,
+   *     (datasource, segmentsMap) -> {
+   *       if (segmentsMap == null) return null;
+   *       else {
+   *         segmentsMap.compute(
+   *           segmentIdParam,
+   *           (segmentId, segmentMetadata) -> {
+   *             // update segmentMetadata
+   *           }
+   *         );
+   *         return segmentsMap;
+   *       }
+   *     }
+   *   );
+   *
    + * + * Readers can simply delegate the locking to the concurrent map and iterate map entries. + */ + private final ConcurrentHashMap> segmentMetadataInfo + = new ConcurrentHashMap<>(); + + // For awaitInitialization. + private final CountDownLatch initialized = new CountDownLatch(1); + + // All mutable segments. + @GuardedBy("lock") + private final TreeSet mutableSegments = new TreeSet<>(SEGMENT_ORDER); + + // Configured context to attach to internally generated queries. + private final InternalQueryConfig internalQueryConfig; + + @GuardedBy("lock") + private boolean refreshImmediately = false; + + @GuardedBy("lock") + private boolean isServerViewInitialized = false; + + /** + * Counts the total number of known segments. This variable is used only for the segments table in the system schema + * to initialize a map with a more proper size when it creates a snapshot. As a result, it doesn't have to be exact, + * and thus there is no concurrency control for this variable. + */ + private int totalSegments = 0; + + /** + * This lock coordinates the access from multiple threads to those variables guarded by this lock. + * Currently, there are 2 threads that can access these variables. + * + * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. + * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceInformation} if necessary + * based on the information collected via timeline callbacks. + */ + protected final Object lock = new Object(); + + // All dataSources that need tables regenerated. + @GuardedBy("lock") + protected final Set dataSourcesNeedingRebuild = new HashSet<>(); + + // All segments that need to be refreshed. + @GuardedBy("lock") + protected final TreeSet segmentsNeedingRefresh = new TreeSet<>(SEGMENT_ORDER); + + public AbstractSegmentMetadataCache( + final QueryLifecycleFactory queryLifecycleFactory, + final TimelineServerView serverView, + final SegmentMetadataCacheConfig config, + final Escalator escalator, + final InternalQueryConfig internalQueryConfig, + final ServiceEmitter emitter + ) + { + this.queryLifecycleFactory = Preconditions.checkNotNull(queryLifecycleFactory, "queryLifecycleFactory"); + Preconditions.checkNotNull(serverView, "serverView"); + this.config = Preconditions.checkNotNull(config, "config"); + this.columnTypeMergePolicy = config.getMetadataColumnTypeMergePolicy(); + this.cacheExec = Execs.singleThreaded("DruidSchema-Cache-%d"); + this.callbackExec = Execs.singleThreaded("DruidSchema-Callback-%d"); + this.escalator = escalator; + this.internalQueryConfig = internalQueryConfig; + this.emitter = emitter; + + initServerViewTimelineCallback(serverView); + } + + private void initServerViewTimelineCallback(final TimelineServerView serverView) + { + serverView.registerTimelineCallback( + callbackExec, + new TimelineServerView.TimelineCallback() + { + @Override + public ServerView.CallbackAction timelineInitialized() + { + synchronized (lock) { + isServerViewInitialized = true; + lock.notifyAll(); + } + + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentAdded(final DruidServerMetadata server, final DataSegment segment) + { + addSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction segmentRemoved(final DataSegment segment) + { + removeSegment(segment); + return ServerView.CallbackAction.CONTINUE; + } + + @Override + public ServerView.CallbackAction serverSegmentRemoved( + final DruidServerMetadata server, + final DataSegment segment + ) + { + removeServerSegment(server, segment); + return ServerView.CallbackAction.CONTINUE; + } + } + ); + } + + private void startCacheExec() + { + cacheExec.submit( + () -> { + long lastRefresh = 0L; + long lastFailure = 0L; + + try { + while (!Thread.currentThread().isInterrupted()) { + final Set segmentsToRefresh = new TreeSet<>(); + final Set dataSourcesToRebuild = new TreeSet<>(); + + try { + synchronized (lock) { + final long nextRefreshNoFuzz = DateTimes + .utc(lastRefresh) + .plus(config.getMetadataRefreshPeriod()) + .getMillis(); + + // Fuzz a bit to spread load out when we have multiple brokers. + final long nextRefresh = nextRefreshNoFuzz + (long) ((nextRefreshNoFuzz - lastRefresh) * 0.10); + + while (true) { + // Do not refresh if it's too soon after a failure (to avoid rapid cycles of failure). + final boolean wasRecentFailure = DateTimes.utc(lastFailure) + .plus(config.getMetadataRefreshPeriod()) + .isAfterNow(); + + if (isServerViewInitialized && + !wasRecentFailure && + (!segmentsNeedingRefresh.isEmpty() || !dataSourcesNeedingRebuild.isEmpty()) && + (refreshImmediately || nextRefresh < System.currentTimeMillis())) { + // We need to do a refresh. Break out of the waiting loop. + break; + } + + // lastFailure != 0L means exceptions happened before and there're some refresh work was not completed. + // so that even if ServerView is initialized, we can't let broker complete initialization. + if (isServerViewInitialized && lastFailure == 0L) { + // Server view is initialized, but we don't need to do a refresh. Could happen if there are + // no segments in the system yet. Just mark us as initialized, then. + initialized.countDown(); + } + + // Wait some more, we'll wake up when it might be time to do another refresh. + lock.wait(Math.max(1, nextRefresh - System.currentTimeMillis())); + } + + segmentsToRefresh.addAll(segmentsNeedingRefresh); + segmentsNeedingRefresh.clear(); + + // Mutable segments need a refresh every period, since new columns could be added dynamically. + segmentsNeedingRefresh.addAll(mutableSegments); + + lastFailure = 0L; + lastRefresh = System.currentTimeMillis(); + refreshImmediately = false; + } + + refresh(segmentsToRefresh, dataSourcesToRebuild); + + initialized.countDown(); + } + catch (InterruptedException e) { + // Fall through. + throw e; + } + catch (Exception e) { + log.warn(e, "Metadata refresh failed, trying again soon."); + + synchronized (lock) { + // Add our segments and dataSources back to their refresh and rebuild lists. + segmentsNeedingRefresh.addAll(segmentsToRefresh); + dataSourcesNeedingRebuild.addAll(dataSourcesToRebuild); + lastFailure = System.currentTimeMillis(); + } + } + } + } + catch (InterruptedException e) { + // Just exit. + } + catch (Throwable e) { + // Throwables that fall out to here (not caught by an inner try/catch) are potentially gnarly, like + // OOMEs. Anyway, let's just emit an alert and stop refreshing metadata. + log.makeAlert(e, "Metadata refresh failed permanently").emit(); + throw e; + } + finally { + log.info("Metadata refresh stopped."); + } + } + ); + } + + @LifecycleStart + public void start() throws InterruptedException + { + log.info("Starting SegmentMetadataCache."); + startCacheExec(); + + if (config.isAwaitInitializationOnStart()) { + final long startMillis = System.currentTimeMillis(); + log.info("%s waiting for initialization.", getClass().getSimpleName()); + awaitInitialization(); + final long endMillis = System.currentTimeMillis(); + log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); + emitter.emit(ServiceMetricEvent.builder().setMetric( + "metadatacache/init/time", + endMillis - startMillis + )); + } + } + + public abstract void refresh(Set segmentsToRefresh, Set dataSourcesToRebuild) throws IOException; + + @LifecycleStop + public void stop() + { + cacheExec.shutdownNow(); + callbackExec.shutdownNow(); + } + + public void awaitInitialization() throws InterruptedException + { + initialized.await(); + } + + public T getDatasource(String name) + { + return tables.get(name); + } + + public Map getDataSourceInformationMap() + { + return ImmutableMap.copyOf(tables); + } + + public Set getDatasourceNames() + { + return tables.keySet(); + } + + @VisibleForTesting + public void addSegment(final DruidServerMetadata server, final DataSegment segment) + { + // Get lock first so that we won't wait in ConcurrentMap.compute(). + synchronized (lock) { + // someday we could hypothetically remove broker special casing, whenever BrokerServerView supports tracking + // broker served segments in the timeline, to ensure that removeSegment the event is triggered accurately + if (server.getType().equals(ServerType.BROKER)) { + // a segment on a broker means a broadcast datasource, skip metadata because we'll also see this segment on the + // historical, however mark the datasource for refresh because it needs to be globalized + markDataSourceAsNeedRebuild(segment.getDataSource()); + } else { + segmentMetadataInfo.compute( + segment.getDataSource(), + (datasource, segmentsMap) -> { + if (segmentsMap == null) { + segmentsMap = new ConcurrentSkipListMap<>(SEGMENT_ORDER); + } + segmentsMap.compute( + segment.getId(), + (segmentId, segmentMetadata) -> { + if (segmentMetadata == null) { + // Unknown segment. + totalSegments++; + // segmentReplicatable is used to determine if segments are served by historical or realtime servers + long isRealtime = server.isSegmentReplicationTarget() ? 0 : 1; + segmentMetadata = AvailableSegmentMetadata + .builder(segment, isRealtime, ImmutableSet.of(server), null, DEFAULT_NUM_ROWS) // Added without needing a refresh + .build(); + markSegmentAsNeedRefresh(segment.getId()); + if (!server.isSegmentReplicationTarget()) { + log.debug("Added new mutable segment [%s].", segment.getId()); + markSegmentAsMutable(segment.getId()); + } else { + log.debug("Added new immutable segment [%s].", segment.getId()); + } + } else { + // We know this segment. + final Set segmentServers = segmentMetadata.getReplicas(); + final ImmutableSet servers = new ImmutableSet.Builder() + .addAll(segmentServers) + .add(server) + .build(); + segmentMetadata = AvailableSegmentMetadata + .from(segmentMetadata) + .withReplicas(servers) + .withRealtime(recomputeIsRealtime(servers)) + .build(); + if (server.isSegmentReplicationTarget()) { + // If a segment shows up on a replicatable (historical) server at any point, then it must be immutable, + // even if it's also available on non-replicatable (realtime) servers. + unmarkSegmentAsMutable(segment.getId()); + log.debug("Segment[%s] has become immutable.", segment.getId()); + } + } + assert segmentMetadata != null; + return segmentMetadata; + } + ); + + return segmentsMap; + } + ); + } + if (!tables.containsKey(segment.getDataSource())) { + refreshImmediately = true; + } + + lock.notifyAll(); + } + } + + @VisibleForTesting + public void removeSegment(final DataSegment segment) + { + // Get lock first so that we won't wait in ConcurrentMap.compute(). + synchronized (lock) { + log.debug("Segment [%s] is gone.", segment.getId()); + + segmentsNeedingRefresh.remove(segment.getId()); + unmarkSegmentAsMutable(segment.getId()); + + segmentMetadataInfo.compute( + segment.getDataSource(), + (dataSource, segmentsMap) -> { + if (segmentsMap == null) { + log.warn("Unknown segment [%s] was removed from the cluster. Ignoring this event.", segment.getId()); + return null; + } else { + if (segmentsMap.remove(segment.getId()) == null) { + log.warn("Unknown segment [%s] was removed from the cluster. Ignoring this event.", segment.getId()); + } else { + totalSegments--; + } + if (segmentsMap.isEmpty()) { + tables.remove(segment.getDataSource()); + log.info("dataSource [%s] no longer exists, all metadata removed.", segment.getDataSource()); + return null; + } else { + markDataSourceAsNeedRebuild(segment.getDataSource()); + return segmentsMap; + } + } + } + ); + + lock.notifyAll(); + } + } + + @VisibleForTesting + public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) + { + // Get lock first so that we won't wait in ConcurrentMap.compute(). + synchronized (lock) { + log.debug("Segment [%s] is gone from server [%s]", segment.getId(), server.getName()); + segmentMetadataInfo.compute( + segment.getDataSource(), + (datasource, knownSegments) -> { + if (knownSegments == null) { + log.warn( + "Unknown segment [%s] is removed from server [%s]. Ignoring this event", + segment.getId(), + server.getHost() + ); + return null; + } + + if (server.getType().equals(ServerType.BROKER)) { + // for brokers, if the segment drops from all historicals before the broker this could be null. + if (!knownSegments.isEmpty()) { + // a segment on a broker means a broadcast datasource, skip metadata because we'll also see this segment on the + // historical, however mark the datasource for refresh because it might no longer be broadcast or something + markDataSourceAsNeedRebuild(segment.getDataSource()); + } + } else { + knownSegments.compute( + segment.getId(), + (segmentId, segmentMetadata) -> { + if (segmentMetadata == null) { + log.warn( + "Unknown segment [%s] is removed from server [%s]. Ignoring this event", + segment.getId(), + server.getHost() + ); + return null; + } else { + final Set segmentServers = segmentMetadata.getReplicas(); + final ImmutableSet servers = FluentIterable + .from(segmentServers) + .filter(Predicates.not(Predicates.equalTo(server))) + .toSet(); + return AvailableSegmentMetadata + .from(segmentMetadata) + .withReplicas(servers) + .withRealtime(recomputeIsRealtime(servers)) + .build(); + } + } + ); + } + if (knownSegments.isEmpty()) { + return null; + } else { + return knownSegments; + } + } + ); + + lock.notifyAll(); + } + } + + private void markSegmentAsNeedRefresh(SegmentId segmentId) + { + synchronized (lock) { + segmentsNeedingRefresh.add(segmentId); + } + } + + private void markSegmentAsMutable(SegmentId segmentId) + { + synchronized (lock) { + mutableSegments.add(segmentId); + } + } + + private void unmarkSegmentAsMutable(SegmentId segmentId) + { + synchronized (lock) { + mutableSegments.remove(segmentId); + } + } + + @VisibleForTesting + public void markDataSourceAsNeedRebuild(String datasource) + { + synchronized (lock) { + dataSourcesNeedingRebuild.add(datasource); + } + } + + /** + * Attempt to refresh "segmentSignatures" for a set of segments. Returns the set of segments actually refreshed, + * which may be a subset of the asked-for set. + */ + @VisibleForTesting + public Set refreshSegments(final Set segments) throws IOException + { + final Set retVal = new HashSet<>(); + + // Organize segments by dataSource. + final Map> segmentMap = new TreeMap<>(); + + for (SegmentId segmentId : segments) { + segmentMap.computeIfAbsent(segmentId.getDataSource(), x -> new TreeSet<>(SEGMENT_ORDER)) + .add(segmentId); + } + + for (Map.Entry> entry : segmentMap.entrySet()) { + final String dataSource = entry.getKey(); + retVal.addAll(refreshSegmentsForDataSource(dataSource, entry.getValue())); + } + + return retVal; + } + + private long recomputeIsRealtime(ImmutableSet servers) + { + if (servers.isEmpty()) { + return 0; + } + final Optional historicalServer = servers + .stream() + // Ideally, this filter should have checked whether it's a broadcast segment loaded in brokers. + // However, we don't current track of the broadcast segments loaded in brokers, so this filter is still valid. + // See addSegment(), removeServerSegment(), and removeSegment() + .filter(metadata -> metadata.getType().equals(ServerType.HISTORICAL)) + .findAny(); + + // if there is any historical server in the replicas, isRealtime flag should be unset + return historicalServer.isPresent() ? 0 : 1; + } + + /** + * Attempt to refresh "segmentSignatures" for a set of segments for a particular dataSource. Returns the set of + * segments actually refreshed, which may be a subset of the asked-for set. + */ + private Set refreshSegmentsForDataSource(final String dataSource, final Set segments) + throws IOException + { + final Stopwatch stopwatch = Stopwatch.createStarted(); + + if (!segments.stream().allMatch(segmentId -> segmentId.getDataSource().equals(dataSource))) { + // Sanity check. We definitely expect this to pass. + throw new ISE("'segments' must all match 'dataSource'!"); + } + + log.debug("Refreshing metadata for dataSource[%s].", dataSource); + + final ServiceMetricEvent.Builder builder = + new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, dataSource); + + emitter.emit(builder.setMetric("metadatacache/refresh/count", segments.size())); + + // Segment id string -> SegmentId object. + final Map segmentIdMap = Maps.uniqueIndex(segments, SegmentId::toString); + + final Set retVal = new HashSet<>(); + final Sequence sequence = runSegmentMetadataQuery( + Iterables.limit(segments, MAX_SEGMENTS_PER_QUERY) + ); + + Yielder yielder = Yielders.each(sequence); + + try { + while (!yielder.isDone()) { + final SegmentAnalysis analysis = yielder.get(); + final SegmentId segmentId = segmentIdMap.get(analysis.getId()); + + if (segmentId == null) { + log.warn("Got analysis for segment [%s] we didn't ask for, ignoring.", analysis.getId()); + } else { + final RowSignature rowSignature = analysisToRowSignature(analysis); + log.debug("Segment[%s] has signature[%s].", segmentId, rowSignature); + segmentMetadataInfo.compute( + dataSource, + (datasourceKey, dataSourceSegments) -> { + if (dataSourceSegments == null) { + // Datasource may have been removed or become unavailable while this refresh was ongoing. + log.warn( + "No segment map found with datasource [%s], skipping refresh of segment [%s]", + datasourceKey, + segmentId + ); + return null; + } else { + dataSourceSegments.compute( + segmentId, + (segmentIdKey, segmentMetadata) -> { + if (segmentMetadata == null) { + log.warn("No segment [%s] found, skipping refresh", segmentId); + return null; + } else { + final AvailableSegmentMetadata updatedSegmentMetadata = AvailableSegmentMetadata + .from(segmentMetadata) + .withRowSignature(rowSignature) + .withNumRows(analysis.getNumRows()) + .build(); + retVal.add(segmentId); + return updatedSegmentMetadata; + } + } + ); + + if (dataSourceSegments.isEmpty()) { + return null; + } else { + return dataSourceSegments; + } + } + } + ); + } + + yielder = yielder.next(null); + } + } + finally { + yielder.close(); + } + + long refreshDurationMillis = stopwatch.elapsed(TimeUnit.MILLISECONDS); + + emitter.emit(builder.setMetric("metadatacache/refresh/time", refreshDurationMillis)); + + log.debug( + "Refreshed metadata for dataSource [%s] in %,d ms (%d segments queried, %d segments left).", + dataSource, + refreshDurationMillis, + retVal.size(), + segments.size() - retVal.size() + ); + + return retVal; + } + + @VisibleForTesting + @Nullable + public RowSignature buildDruidTable(final String dataSource) + { + ConcurrentSkipListMap segmentsMap = segmentMetadataInfo.get(dataSource); + + // Preserve order. + final Map columnTypes = new LinkedHashMap<>(); + + if (segmentsMap != null && !segmentsMap.isEmpty()) { + for (AvailableSegmentMetadata availableSegmentMetadata : segmentsMap.values()) { + final RowSignature rowSignature = availableSegmentMetadata.getRowSignature(); + if (rowSignature != null) { + for (String column : rowSignature.getColumnNames()) { + final ColumnType columnType = + rowSignature.getColumnType(column) + .orElseThrow(() -> new ISE("Encountered null type for column [%s]", column)); + + columnTypes.compute(column, (c, existingType) -> columnTypeMergePolicy.merge(existingType, columnType)); + } + } + } + } else { + // table has no segments + return null; + } + + final RowSignature.Builder builder = RowSignature.builder(); + columnTypes.forEach(builder::add); + + return builder.build(); + } + + public Map getSegmentMetadataSnapshot() + { + final Map segmentMetadata = Maps.newHashMapWithExpectedSize(totalSegments); + for (ConcurrentSkipListMap val : segmentMetadataInfo.values()) { + segmentMetadata.putAll(val); + } + return segmentMetadata; + } + + @Nullable + public AvailableSegmentMetadata getAvailableSegmentMetadata(String datasource, SegmentId segmentId) + { + if (!segmentMetadataInfo.containsKey(datasource)) { + return null; + } + return segmentMetadataInfo.get(datasource).get(segmentId); + } + + /** + * Returns total number of segments. This method doesn't use the lock intentionally to avoid expensive contention. + * As a result, the returned value might be inexact. + */ + public int getTotalSegments() + { + return totalSegments; + } + + @VisibleForTesting + public TreeSet getSegmentsNeedingRefresh() + { + synchronized (lock) { + return segmentsNeedingRefresh; + } + } + + @VisibleForTesting + public TreeSet getMutableSegments() + { + synchronized (lock) { + return mutableSegments; + } + } + + @VisibleForTesting + public Set getDataSourcesNeedingRebuild() + { + synchronized (lock) { + return dataSourcesNeedingRebuild; + } + } + + /** + * Execute a SegmentMetadata query and return a {@link Sequence} of {@link SegmentAnalysis}. + * + * @param segments Iterable of {@link SegmentId} objects that are subject of the SegmentMetadata query. + * @return {@link Sequence} of {@link SegmentAnalysis} objects + */ + @VisibleForTesting + public Sequence runSegmentMetadataQuery( + final Iterable segments + ) + { + // Sanity check: getOnlyElement of a set, to ensure all segments have the same dataSource. + final String dataSource = Iterables.getOnlyElement( + StreamSupport.stream(segments.spliterator(), false) + .map(SegmentId::getDataSource).collect(Collectors.toSet()) + ); + + final MultipleSpecificSegmentSpec querySegmentSpec = new MultipleSpecificSegmentSpec( + StreamSupport.stream(segments.spliterator(), false) + .map(SegmentId::toDescriptor).collect(Collectors.toList()) + ); + + final SegmentMetadataQuery segmentMetadataQuery = new SegmentMetadataQuery( + new TableDataSource(dataSource), + querySegmentSpec, + new AllColumnIncluderator(), + false, + // disable the parallel merge because we don't care about the merge and don't want to consume its resources + QueryContexts.override( + internalQueryConfig.getContext(), + QueryContexts.BROKER_PARALLEL_MERGE_KEY, + false + ), + EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), + false, + null, + null // we don't care about merging strategy because merge is false + ); + + return queryLifecycleFactory + .factorize() + .runSimple(segmentMetadataQuery, escalator.createEscalatedAuthenticationResult(), Access.OK).getResults(); + } + + @VisibleForTesting + static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) + { + final RowSignature.Builder rowSignatureBuilder = RowSignature.builder(); + for (Map.Entry entry : analysis.getColumns().entrySet()) { + if (entry.getValue().isError()) { + // Skip columns with analysis errors. + continue; + } + + ColumnType valueType = entry.getValue().getTypeSignature(); + + // this shouldn't happen, but if it does, first try to fall back to legacy type information field in case + // standard upgrade order was not followed for 0.22 to 0.23+, and if that also fails, then assume types are some + // flavor of COMPLEX. + if (valueType == null) { + // at some point in the future this can be simplified to the contents of the catch clause here, once the + // likelyhood of upgrading from some version lower than 0.23 is low + try { + valueType = ColumnType.fromString(entry.getValue().getType()); + if (valueType == null) { + valueType = ColumnType.ofComplex(entry.getValue().getType()); + } + } + catch (IllegalArgumentException ignored) { + valueType = ColumnType.UNKNOWN_COMPLEX; + } + } + + rowSignatureBuilder.add(entry.getKey(), valueType); + } + return ROW_SIGNATURE_INTERNER.intern(rowSignatureBuilder.build()); + } + + /** + * This method is not thread-safe and must be used only in unit tests. + */ + @VisibleForTesting + void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) + { + final ConcurrentSkipListMap dataSourceSegments = segmentMetadataInfo + .computeIfAbsent( + segmentId.getDataSource(), + k -> new ConcurrentSkipListMap<>(SEGMENT_ORDER) + ); + if (dataSourceSegments.put(segmentId, availableSegmentMetadata) == null) { + totalSegments++; + } + } + + /** + * This is a helper method for unit tests to emulate heavy work done with {@link #lock}. + * It must be used only in unit tests. + */ + @VisibleForTesting + void doInLock(Runnable runnable) + { + synchronized (lock) { + runnable.run(); + } + } + + /** + * ColumnTypeMergePolicy defines the rules of which type to use when faced with the possibility of different types + * for the same column from segment to segment. It is used to help compute a {@link RowSignature} for a table in + * Druid based on the segment metadata of all segments, merging the types of each column encountered to end up with + * a single type to represent it globally. + */ + @FunctionalInterface + public interface ColumnTypeMergePolicy + { + ColumnType merge(ColumnType existingType, ColumnType newType); + + @JsonCreator + static ColumnTypeMergePolicy fromString(String type) + { + if (LeastRestrictiveTypeMergePolicy.NAME.equalsIgnoreCase(type)) { + return LeastRestrictiveTypeMergePolicy.INSTANCE; + } + if (FirstTypeMergePolicy.NAME.equalsIgnoreCase(type)) { + return FirstTypeMergePolicy.INSTANCE; + } + throw new IAE("Unknown type [%s]", type); + } + } + + /** + * Classic logic, we use the first type we encounter. This policy is effectively 'newest first' because we iterated + * segments starting from the most recent time chunk, so this typically results in the most recently used type being + * chosen, at least for systems that are continuously updated with 'current' data. + * + * Since {@link ColumnTypeMergePolicy} are used to compute the SQL schema, at least in systems using SQL schemas which + * are partially or fully computed by this cache, this merge policy can result in query time errors if incompatible + * types are mixed if the chosen type is more restrictive than the types of some segments. If data is likely to vary + * in type across segments, consider using {@link LeastRestrictiveTypeMergePolicy} instead. + */ + public static class FirstTypeMergePolicy implements ColumnTypeMergePolicy + { + public static final String NAME = "latestInterval"; + private static final FirstTypeMergePolicy INSTANCE = new FirstTypeMergePolicy(); + + @Override + public ColumnType merge(ColumnType existingType, ColumnType newType) + { + if (existingType == null) { + return newType; + } + if (newType == null) { + return existingType; + } + // if any are json, are all json + if (ColumnType.NESTED_DATA.equals(newType) || ColumnType.NESTED_DATA.equals(existingType)) { + return ColumnType.NESTED_DATA; + } + // "existing type" is the 'newest' type, since we iterate the segments list by newest start time + return existingType; + } + + @Override + public int hashCode() + { + return Objects.hash(NAME); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + return o != null && getClass() == o.getClass(); + } + + @Override + public String toString() + { + return NAME; + } + } + + /** + * Resolves types using {@link ColumnType#leastRestrictiveType(ColumnType, ColumnType)} to find the ColumnType that + * can best represent all data contained across all segments. + */ + public static class LeastRestrictiveTypeMergePolicy implements ColumnTypeMergePolicy + { + public static final String NAME = "leastRestrictive"; + private static final LeastRestrictiveTypeMergePolicy INSTANCE = new LeastRestrictiveTypeMergePolicy(); + + @Override + public ColumnType merge(ColumnType existingType, ColumnType newType) + { + try { + return ColumnType.leastRestrictiveType(existingType, newType); + } + catch (Types.IncompatibleTypeException incompatibleTypeException) { + // fall back to first encountered type if they are not compatible for some reason + return FirstTypeMergePolicy.INSTANCE.merge(existingType, newType); + } + } + + @Override + public int hashCode() + { + return Objects.hash(NAME); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + return o != null && getClass() == o.getClass(); + } + + @Override + public String toString() + { + return NAME; + } + } +} diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java index 2ff4640f1297..83c67532d6e2 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java @@ -19,381 +19,48 @@ package org.apache.druid.segment.metadata; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicates; -import com.google.common.base.Stopwatch; -import com.google.common.collect.FluentIterable; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Interner; -import com.google.common.collect.Interners; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.google.errorprone.annotations.concurrent.GuardedBy; import com.google.inject.Inject; import org.apache.druid.client.InternalQueryConfig; -import org.apache.druid.client.ServerView; import org.apache.druid.client.TimelineServerView; import org.apache.druid.guice.ManageLifecycle; -import org.apache.druid.java.util.common.DateTimes; -import org.apache.druid.java.util.common.IAE; -import org.apache.druid.java.util.common.ISE; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.common.guava.Sequence; -import org.apache.druid.java.util.common.guava.Yielder; -import org.apache.druid.java.util.common.guava.Yielders; -import org.apache.druid.java.util.common.lifecycle.LifecycleStart; -import org.apache.druid.java.util.common.lifecycle.LifecycleStop; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; -import org.apache.druid.query.DruidMetrics; -import org.apache.druid.query.QueryContexts; -import org.apache.druid.query.TableDataSource; -import org.apache.druid.query.metadata.metadata.AllColumnIncluderator; -import org.apache.druid.query.metadata.metadata.ColumnAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentAnalysis; -import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; -import org.apache.druid.query.spec.MultipleSpecificSegmentSpec; -import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; -import org.apache.druid.segment.column.Types; import org.apache.druid.server.QueryLifecycleFactory; -import org.apache.druid.server.coordination.DruidServerMetadata; -import org.apache.druid.server.coordination.ServerType; -import org.apache.druid.server.security.Access; import org.apache.druid.server.security.Escalator; -import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; -import javax.annotation.Nullable; import java.io.IOException; -import java.util.Comparator; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; /** * Coordinator-side cache of segment metadata that combines segments to identify * datasources. The cache provides metadata about a dataSource, see {@link DataSourceInformation}. */ @ManageLifecycle -public class SegmentMetadataCache +public class SegmentMetadataCache extends AbstractSegmentMetadataCache { - // Newest segments first, so they override older ones. - private static final Comparator SEGMENT_ORDER = Comparator - .comparing((SegmentId segmentId) -> segmentId.getInterval().getStart()) - .reversed() - .thenComparing(Function.identity()); - - private static final EmittingLogger log = new EmittingLogger(SegmentMetadataCache.class); - private static final int MAX_SEGMENTS_PER_QUERY = 15000; - private static final long DEFAULT_NUM_ROWS = 0; - private static final Interner ROW_SIGNATURE_INTERNER = Interners.newWeakInterner(); - private final QueryLifecycleFactory queryLifecycleFactory; - private final SegmentMetadataCacheConfig config; - // Escalator, so we can attach an authentication result to queries we generate. - private final Escalator escalator; - - private final ExecutorService cacheExec; - private final ExecutorService callbackExec; - private final ServiceEmitter emitter; - private final ColumnTypeMergePolicy columnTypeMergePolicy; - - /** - * Map of dataSource -> DataSourceInformation - * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. - */ - protected final ConcurrentMap tables = new ConcurrentHashMap<>(); - - /** - * DataSource -> Segment -> AvailableSegmentMetadata(contains RowSignature) for that segment. - * Use SortedMap for segments so they are merged in deterministic order, from older to newer. - * - * This map is updated by these two threads. - * - * - {@link #callbackExec} can update it in {@link #addSegment}, {@link #removeServerSegment}, - * and {@link #removeSegment}. - * - {@link #cacheExec} can update it in {@link #refreshSegmentsForDataSource}. - * - * While it is being updated, this map is read by these two types of thread. - * - * - {@link #cacheExec} can iterate all {@link AvailableSegmentMetadata}s per datasource. - * See {@link #buildDruidTable}. - * - Query threads can create a snapshot of the entire map for processing queries on the system table. - * See {@link #getSegmentMetadataSnapshot()}. - * - * As the access pattern of this map is read-intensive, we should minimize the contention between writers and readers. - * Since there are two threads that can update this map at the same time, those writers should lock the inner map - * first and then lock the entry before it updates segment metadata. This can be done using - * {@link ConcurrentMap#compute} as below. Note that, if you need to update the variables guarded by {@link #lock} - * inside of compute(), you should get the lock before calling compute() to keep the function executed in compute() - * not expensive. - * - * 
    -   *   segmentMedataInfo.compute(
-   *     datasourceParam,
-   *     (datasource, segmentsMap) -> {
-   *       if (segmentsMap == null) return null;
-   *       else {
-   *         segmentsMap.compute(
-   *           segmentIdParam,
-   *           (segmentId, segmentMetadata) -> {
-   *             // update segmentMetadata
-   *           }
-   *         );
-   *         return segmentsMap;
-   *       }
-   *     }
-   *   );
-   *
    - * - * Readers can simply delegate the locking to the concurrent map and iterate map entries. - */ - private final ConcurrentHashMap> segmentMetadataInfo - = new ConcurrentHashMap<>(); - - // For awaitInitialization. - private final CountDownLatch initialized = new CountDownLatch(1); - - // All mutable segments. - @GuardedBy("lock") - private final TreeSet mutableSegments = new TreeSet<>(SEGMENT_ORDER); - - // Configured context to attach to internally generated queries. - private final InternalQueryConfig internalQueryConfig; - - @GuardedBy("lock") - private boolean refreshImmediately = false; - - @GuardedBy("lock") - private boolean isServerViewInitialized = false; - - /** - * Counts the total number of known segments. This variable is used only for the segments table in the system schema - * to initialize a map with a more proper size when it creates a snapshot. As a result, it doesn't have to be exact, - * and thus there is no concurrency control for this variable. - */ - private int totalSegments = 0; - - /** - * This lock coordinates the access from multiple threads to those variables guarded by this lock. - * Currently, there are 2 threads that can access these variables. - * - * - {@link #callbackExec} executes the timeline callbacks whenever BrokerServerView changes. - * - {@link #cacheExec} periodically refreshes segment metadata and {@link DataSourceInformation} if necessary - * based on the information collected via timeline callbacks. - */ - protected final Object lock = new Object(); - - // All dataSources that need tables regenerated. - @GuardedBy("lock") - protected final Set dataSourcesNeedingRebuild = new HashSet<>(); - - // All segments that need to be refreshed. - @GuardedBy("lock") - protected final TreeSet segmentsNeedingRefresh = new TreeSet<>(SEGMENT_ORDER); + private static final EmittingLogger log = new EmittingLogger(AbstractSegmentMetadataCache.class); @Inject public SegmentMetadataCache( - final QueryLifecycleFactory queryLifecycleFactory, - final TimelineServerView serverView, - final SegmentMetadataCacheConfig config, - final Escalator escalator, - final InternalQueryConfig internalQueryConfig, - final ServiceEmitter emitter + QueryLifecycleFactory queryLifecycleFactory, + TimelineServerView serverView, + SegmentMetadataCacheConfig config, + Escalator escalator, + InternalQueryConfig internalQueryConfig, + ServiceEmitter emitter ) { - this.queryLifecycleFactory = Preconditions.checkNotNull(queryLifecycleFactory, "queryLifecycleFactory"); - Preconditions.checkNotNull(serverView, "serverView"); - this.config = Preconditions.checkNotNull(config, "config"); - this.columnTypeMergePolicy = config.getMetadataColumnTypeMergePolicy(); - this.cacheExec = Execs.singleThreaded("DruidSchema-Cache-%d"); - this.callbackExec = Execs.singleThreaded("DruidSchema-Callback-%d"); - this.escalator = escalator; - this.internalQueryConfig = internalQueryConfig; - this.emitter = emitter; - - initServerViewTimelineCallback(serverView); - } - - private void initServerViewTimelineCallback(final TimelineServerView serverView) - { - serverView.registerTimelineCallback( - callbackExec, - new TimelineServerView.TimelineCallback() - { - @Override - public ServerView.CallbackAction timelineInitialized() - { - synchronized (lock) { - isServerViewInitialized = true; - lock.notifyAll(); - } - - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentAdded(final DruidServerMetadata server, final DataSegment segment) - { - addSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction segmentRemoved(final DataSegment segment) - { - removeSegment(segment); - return ServerView.CallbackAction.CONTINUE; - } - - @Override - public ServerView.CallbackAction serverSegmentRemoved( - final DruidServerMetadata server, - final DataSegment segment - ) - { - removeServerSegment(server, segment); - return ServerView.CallbackAction.CONTINUE; - } - } - ); - } - - private void startCacheExec() - { - cacheExec.submit( - () -> { - long lastRefresh = 0L; - long lastFailure = 0L; - - try { - while (!Thread.currentThread().isInterrupted()) { - final Set segmentsToRefresh = new TreeSet<>(); - final Set dataSourcesToRebuild = new TreeSet<>(); - - try { - synchronized (lock) { - final long nextRefreshNoFuzz = DateTimes - .utc(lastRefresh) - .plus(config.getMetadataRefreshPeriod()) - .getMillis(); - - // Fuzz a bit to spread load out when we have multiple brokers. - final long nextRefresh = nextRefreshNoFuzz + (long) ((nextRefreshNoFuzz - lastRefresh) * 0.10); - - while (true) { - // Do not refresh if it's too soon after a failure (to avoid rapid cycles of failure). - final boolean wasRecentFailure = DateTimes.utc(lastFailure) - .plus(config.getMetadataRefreshPeriod()) - .isAfterNow(); - - if (isServerViewInitialized && - !wasRecentFailure && - (!segmentsNeedingRefresh.isEmpty() || !dataSourcesNeedingRebuild.isEmpty()) && - (refreshImmediately || nextRefresh < System.currentTimeMillis())) { - // We need to do a refresh. Break out of the waiting loop. - break; - } - - // lastFailure != 0L means exceptions happened before and there're some refresh work was not completed. - // so that even if ServerView is initialized, we can't let broker complete initialization. - if (isServerViewInitialized && lastFailure == 0L) { - // Server view is initialized, but we don't need to do a refresh. Could happen if there are - // no segments in the system yet. Just mark us as initialized, then. - initialized.countDown(); - } - - // Wait some more, we'll wake up when it might be time to do another refresh. - lock.wait(Math.max(1, nextRefresh - System.currentTimeMillis())); - } - - segmentsToRefresh.addAll(segmentsNeedingRefresh); - segmentsNeedingRefresh.clear(); - - // Mutable segments need a refresh every period, since new columns could be added dynamically. - segmentsNeedingRefresh.addAll(mutableSegments); - - lastFailure = 0L; - lastRefresh = System.currentTimeMillis(); - refreshImmediately = false; - } - - refresh(segmentsToRefresh, dataSourcesToRebuild); - - initialized.countDown(); - } - catch (InterruptedException e) { - // Fall through. - throw e; - } - catch (Exception e) { - log.warn(e, "Metadata refresh failed, trying again soon."); - - synchronized (lock) { - // Add our segments and dataSources back to their refresh and rebuild lists. - segmentsNeedingRefresh.addAll(segmentsToRefresh); - dataSourcesNeedingRebuild.addAll(dataSourcesToRebuild); - lastFailure = System.currentTimeMillis(); - } - } - } - } - catch (InterruptedException e) { - // Just exit. - } - catch (Throwable e) { - // Throwables that fall out to here (not caught by an inner try/catch) are potentially gnarly, like - // OOMEs. Anyway, let's just emit an alert and stop refreshing metadata. - log.makeAlert(e, "Metadata refresh failed permanently").emit(); - throw e; - } - finally { - log.info("Metadata refresh stopped."); - } - } - ); - } - - @LifecycleStart - public void start() throws InterruptedException - { - log.info("Starting SegmentMetadataCache."); - startCacheExec(); - - if (config.isAwaitInitializationOnStart()) { - final long startMillis = System.currentTimeMillis(); - log.info("%s waiting for initialization.", getClass().getSimpleName()); - awaitInitialization(); - final long endMillis = System.currentTimeMillis(); - log.info("%s initialized in [%,d] ms.", getClass().getSimpleName(), endMillis - startMillis); - emitter.emit(ServiceMetricEvent.builder().setMetric( - "metadatacache/init/time", - endMillis - startMillis - )); - } + super(queryLifecycleFactory, serverView, config, escalator, internalQueryConfig, emitter); } - @VisibleForTesting + /** + * Fires SegmentMetadataQuery to fetch schema information for each segment in the refresh list. + * The schema information for individual segments is combined to construct a table schema, which is then cached. + */ + @Override public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException { // Refresh the segments. @@ -411,13 +78,14 @@ public void refresh(final Set segmentsToRefresh, final Set da // Rebuild the dataSources. for (String dataSource : dataSourcesToRebuild) { - final DataSourceInformation druidTable = buildDruidTable(dataSource); - if (druidTable == null) { + final RowSignature rowSignature = buildDruidTable(dataSource); + if (rowSignature == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } - final DataSourceInformation oldTable = tables.put(dataSource, (T) druidTable); + DataSourceInformation druidTable = new DataSourceInformation(dataSource, rowSignature); + final DataSourceInformation oldTable = tables.put(dataSource, druidTable); if (oldTable == null || !oldTable.getRowSignature().equals(druidTable.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); } else { @@ -425,690 +93,4 @@ public void refresh(final Set segmentsToRefresh, final Set da } } } - - @LifecycleStop - public void stop() - { - cacheExec.shutdownNow(); - callbackExec.shutdownNow(); - } - - public void awaitInitialization() throws InterruptedException - { - initialized.await(); - } - - public T getDatasource(String name) - { - return tables.get(name); - } - - public Map getDataSourceInformationMap() - { - return ImmutableMap.copyOf(tables); - } - - public Set getDatasourceNames() - { - return tables.keySet(); - } - - @VisibleForTesting - public void addSegment(final DruidServerMetadata server, final DataSegment segment) - { - // Get lock first so that we won't wait in ConcurrentMap.compute(). - synchronized (lock) { - // someday we could hypothetically remove broker special casing, whenever BrokerServerView supports tracking - // broker served segments in the timeline, to ensure that removeSegment the event is triggered accurately - if (server.getType().equals(ServerType.BROKER)) { - // a segment on a broker means a broadcast datasource, skip metadata because we'll also see this segment on the - // historical, however mark the datasource for refresh because it needs to be globalized - markDataSourceAsNeedRebuild(segment.getDataSource()); - } else { - segmentMetadataInfo.compute( - segment.getDataSource(), - (datasource, segmentsMap) -> { - if (segmentsMap == null) { - segmentsMap = new ConcurrentSkipListMap<>(SEGMENT_ORDER); - } - segmentsMap.compute( - segment.getId(), - (segmentId, segmentMetadata) -> { - if (segmentMetadata == null) { - // Unknown segment. - totalSegments++; - // segmentReplicatable is used to determine if segments are served by historical or realtime servers - long isRealtime = server.isSegmentReplicationTarget() ? 0 : 1; - segmentMetadata = AvailableSegmentMetadata - .builder(segment, isRealtime, ImmutableSet.of(server), null, DEFAULT_NUM_ROWS) // Added without needing a refresh - .build(); - markSegmentAsNeedRefresh(segment.getId()); - if (!server.isSegmentReplicationTarget()) { - log.debug("Added new mutable segment [%s].", segment.getId()); - markSegmentAsMutable(segment.getId()); - } else { - log.debug("Added new immutable segment [%s].", segment.getId()); - } - } else { - // We know this segment. - final Set segmentServers = segmentMetadata.getReplicas(); - final ImmutableSet servers = new ImmutableSet.Builder() - .addAll(segmentServers) - .add(server) - .build(); - segmentMetadata = AvailableSegmentMetadata - .from(segmentMetadata) - .withReplicas(servers) - .withRealtime(recomputeIsRealtime(servers)) - .build(); - if (server.isSegmentReplicationTarget()) { - // If a segment shows up on a replicatable (historical) server at any point, then it must be immutable, - // even if it's also available on non-replicatable (realtime) servers. - unmarkSegmentAsMutable(segment.getId()); - log.debug("Segment[%s] has become immutable.", segment.getId()); - } - } - assert segmentMetadata != null; - return segmentMetadata; - } - ); - - return segmentsMap; - } - ); - } - if (!tables.containsKey(segment.getDataSource())) { - refreshImmediately = true; - } - - lock.notifyAll(); - } - } - - @VisibleForTesting - public void removeSegment(final DataSegment segment) - { - // Get lock first so that we won't wait in ConcurrentMap.compute(). - synchronized (lock) { - log.debug("Segment [%s] is gone.", segment.getId()); - - segmentsNeedingRefresh.remove(segment.getId()); - unmarkSegmentAsMutable(segment.getId()); - - segmentMetadataInfo.compute( - segment.getDataSource(), - (dataSource, segmentsMap) -> { - if (segmentsMap == null) { - log.warn("Unknown segment [%s] was removed from the cluster. Ignoring this event.", segment.getId()); - return null; - } else { - if (segmentsMap.remove(segment.getId()) == null) { - log.warn("Unknown segment [%s] was removed from the cluster. Ignoring this event.", segment.getId()); - } else { - totalSegments--; - } - if (segmentsMap.isEmpty()) { - tables.remove(segment.getDataSource()); - log.info("dataSource [%s] no longer exists, all metadata removed.", segment.getDataSource()); - return null; - } else { - markDataSourceAsNeedRebuild(segment.getDataSource()); - return segmentsMap; - } - } - } - ); - - lock.notifyAll(); - } - } - - @VisibleForTesting - public void removeServerSegment(final DruidServerMetadata server, final DataSegment segment) - { - // Get lock first so that we won't wait in ConcurrentMap.compute(). - synchronized (lock) { - log.debug("Segment [%s] is gone from server [%s]", segment.getId(), server.getName()); - segmentMetadataInfo.compute( - segment.getDataSource(), - (datasource, knownSegments) -> { - if (knownSegments == null) { - log.warn( - "Unknown segment [%s] is removed from server [%s]. Ignoring this event", - segment.getId(), - server.getHost() - ); - return null; - } - - if (server.getType().equals(ServerType.BROKER)) { - // for brokers, if the segment drops from all historicals before the broker this could be null. - if (!knownSegments.isEmpty()) { - // a segment on a broker means a broadcast datasource, skip metadata because we'll also see this segment on the - // historical, however mark the datasource for refresh because it might no longer be broadcast or something - markDataSourceAsNeedRebuild(segment.getDataSource()); - } - } else { - knownSegments.compute( - segment.getId(), - (segmentId, segmentMetadata) -> { - if (segmentMetadata == null) { - log.warn( - "Unknown segment [%s] is removed from server [%s]. Ignoring this event", - segment.getId(), - server.getHost() - ); - return null; - } else { - final Set segmentServers = segmentMetadata.getReplicas(); - final ImmutableSet servers = FluentIterable - .from(segmentServers) - .filter(Predicates.not(Predicates.equalTo(server))) - .toSet(); - return AvailableSegmentMetadata - .from(segmentMetadata) - .withReplicas(servers) - .withRealtime(recomputeIsRealtime(servers)) - .build(); - } - } - ); - } - if (knownSegments.isEmpty()) { - return null; - } else { - return knownSegments; - } - } - ); - - lock.notifyAll(); - } - } - - private void markSegmentAsNeedRefresh(SegmentId segmentId) - { - synchronized (lock) { - segmentsNeedingRefresh.add(segmentId); - } - } - - private void markSegmentAsMutable(SegmentId segmentId) - { - synchronized (lock) { - mutableSegments.add(segmentId); - } - } - - private void unmarkSegmentAsMutable(SegmentId segmentId) - { - synchronized (lock) { - mutableSegments.remove(segmentId); - } - } - - @VisibleForTesting - public void markDataSourceAsNeedRebuild(String datasource) - { - synchronized (lock) { - dataSourcesNeedingRebuild.add(datasource); - } - } - - /** - * Attempt to refresh "segmentSignatures" for a set of segments. Returns the set of segments actually refreshed, - * which may be a subset of the asked-for set. - */ - @VisibleForTesting - public Set refreshSegments(final Set segments) throws IOException - { - final Set retVal = new HashSet<>(); - - // Organize segments by dataSource. - final Map> segmentMap = new TreeMap<>(); - - for (SegmentId segmentId : segments) { - segmentMap.computeIfAbsent(segmentId.getDataSource(), x -> new TreeSet<>(SEGMENT_ORDER)) - .add(segmentId); - } - - for (Map.Entry> entry : segmentMap.entrySet()) { - final String dataSource = entry.getKey(); - retVal.addAll(refreshSegmentsForDataSource(dataSource, entry.getValue())); - } - - return retVal; - } - - private long recomputeIsRealtime(ImmutableSet servers) - { - if (servers.isEmpty()) { - return 0; - } - final Optional historicalServer = servers - .stream() - // Ideally, this filter should have checked whether it's a broadcast segment loaded in brokers. - // However, we don't current track of the broadcast segments loaded in brokers, so this filter is still valid. - // See addSegment(), removeServerSegment(), and removeSegment() - .filter(metadata -> metadata.getType().equals(ServerType.HISTORICAL)) - .findAny(); - - // if there is any historical server in the replicas, isRealtime flag should be unset - return historicalServer.isPresent() ? 0 : 1; - } - - /** - * Attempt to refresh "segmentSignatures" for a set of segments for a particular dataSource. Returns the set of - * segments actually refreshed, which may be a subset of the asked-for set. - */ - private Set refreshSegmentsForDataSource(final String dataSource, final Set segments) - throws IOException - { - final Stopwatch stopwatch = Stopwatch.createStarted(); - - if (!segments.stream().allMatch(segmentId -> segmentId.getDataSource().equals(dataSource))) { - // Sanity check. We definitely expect this to pass. - throw new ISE("'segments' must all match 'dataSource'!"); - } - - log.debug("Refreshing metadata for dataSource[%s].", dataSource); - - final ServiceMetricEvent.Builder builder = - new ServiceMetricEvent.Builder().setDimension(DruidMetrics.DATASOURCE, dataSource); - - emitter.emit(builder.setMetric("metadatacache/refresh/count", segments.size())); - - // Segment id string -> SegmentId object. - final Map segmentIdMap = Maps.uniqueIndex(segments, SegmentId::toString); - - final Set retVal = new HashSet<>(); - final Sequence sequence = runSegmentMetadataQuery( - Iterables.limit(segments, MAX_SEGMENTS_PER_QUERY) - ); - - Yielder yielder = Yielders.each(sequence); - - try { - while (!yielder.isDone()) { - final SegmentAnalysis analysis = yielder.get(); - final SegmentId segmentId = segmentIdMap.get(analysis.getId()); - - if (segmentId == null) { - log.warn("Got analysis for segment [%s] we didn't ask for, ignoring.", analysis.getId()); - } else { - final RowSignature rowSignature = analysisToRowSignature(analysis); - log.debug("Segment[%s] has signature[%s].", segmentId, rowSignature); - segmentMetadataInfo.compute( - dataSource, - (datasourceKey, dataSourceSegments) -> { - if (dataSourceSegments == null) { - // Datasource may have been removed or become unavailable while this refresh was ongoing. - log.warn( - "No segment map found with datasource [%s], skipping refresh of segment [%s]", - datasourceKey, - segmentId - ); - return null; - } else { - dataSourceSegments.compute( - segmentId, - (segmentIdKey, segmentMetadata) -> { - if (segmentMetadata == null) { - log.warn("No segment [%s] found, skipping refresh", segmentId); - return null; - } else { - final AvailableSegmentMetadata updatedSegmentMetadata = AvailableSegmentMetadata - .from(segmentMetadata) - .withRowSignature(rowSignature) - .withNumRows(analysis.getNumRows()) - .build(); - retVal.add(segmentId); - return updatedSegmentMetadata; - } - } - ); - - if (dataSourceSegments.isEmpty()) { - return null; - } else { - return dataSourceSegments; - } - } - } - ); - } - - yielder = yielder.next(null); - } - } - finally { - yielder.close(); - } - - long refreshDurationMillis = stopwatch.elapsed(TimeUnit.MILLISECONDS); - - emitter.emit(builder.setMetric("metadatacache/refresh/time", refreshDurationMillis)); - - log.debug( - "Refreshed metadata for dataSource [%s] in %,d ms (%d segments queried, %d segments left).", - dataSource, - refreshDurationMillis, - retVal.size(), - segments.size() - retVal.size() - ); - - return retVal; - } - - @VisibleForTesting - @Nullable - public DataSourceInformation buildDruidTable(final String dataSource) - { - ConcurrentSkipListMap segmentsMap = segmentMetadataInfo.get(dataSource); - - // Preserve order. - final Map columnTypes = new LinkedHashMap<>(); - - if (segmentsMap != null && !segmentsMap.isEmpty()) { - for (AvailableSegmentMetadata availableSegmentMetadata : segmentsMap.values()) { - final RowSignature rowSignature = availableSegmentMetadata.getRowSignature(); - if (rowSignature != null) { - for (String column : rowSignature.getColumnNames()) { - final ColumnType columnType = - rowSignature.getColumnType(column) - .orElseThrow(() -> new ISE("Encountered null type for column [%s]", column)); - - columnTypes.compute(column, (c, existingType) -> columnTypeMergePolicy.merge(existingType, columnType)); - } - } - } - } else { - // table has no segments - return null; - } - - final RowSignature.Builder builder = RowSignature.builder(); - columnTypes.forEach(builder::add); - - return new DataSourceInformation(dataSource, builder.build()); - } - - public Map getSegmentMetadataSnapshot() - { - final Map segmentMetadata = Maps.newHashMapWithExpectedSize(totalSegments); - for (ConcurrentSkipListMap val : segmentMetadataInfo.values()) { - segmentMetadata.putAll(val); - } - return segmentMetadata; - } - - @Nullable - public AvailableSegmentMetadata getAvailableSegmentMetadata(String datasource, SegmentId segmentId) - { - if (!segmentMetadataInfo.containsKey(datasource)) { - return null; - } - return segmentMetadataInfo.get(datasource).get(segmentId); - } - - /** - * Returns total number of segments. This method doesn't use the lock intentionally to avoid expensive contention. - * As a result, the returned value might be inexact. - */ - public int getTotalSegments() - { - return totalSegments; - } - - @VisibleForTesting - public TreeSet getSegmentsNeedingRefresh() - { - synchronized (lock) { - return segmentsNeedingRefresh; - } - } - - @VisibleForTesting - public TreeSet getMutableSegments() - { - synchronized (lock) { - return mutableSegments; - } - } - - @VisibleForTesting - public Set getDataSourcesNeedingRebuild() - { - synchronized (lock) { - return dataSourcesNeedingRebuild; - } - } - - /** - * Execute a SegmentMetadata query and return a {@link Sequence} of {@link SegmentAnalysis}. - * - * @param segments Iterable of {@link SegmentId} objects that are subject of the SegmentMetadata query. - * @return {@link Sequence} of {@link SegmentAnalysis} objects - */ - @VisibleForTesting - public Sequence runSegmentMetadataQuery( - final Iterable segments - ) - { - // Sanity check: getOnlyElement of a set, to ensure all segments have the same dataSource. - final String dataSource = Iterables.getOnlyElement( - StreamSupport.stream(segments.spliterator(), false) - .map(SegmentId::getDataSource).collect(Collectors.toSet()) - ); - - final MultipleSpecificSegmentSpec querySegmentSpec = new MultipleSpecificSegmentSpec( - StreamSupport.stream(segments.spliterator(), false) - .map(SegmentId::toDescriptor).collect(Collectors.toList()) - ); - - final SegmentMetadataQuery segmentMetadataQuery = new SegmentMetadataQuery( - new TableDataSource(dataSource), - querySegmentSpec, - new AllColumnIncluderator(), - false, - // disable the parallel merge because we don't care about the merge and don't want to consume its resources - QueryContexts.override( - internalQueryConfig.getContext(), - QueryContexts.BROKER_PARALLEL_MERGE_KEY, - false - ), - EnumSet.noneOf(SegmentMetadataQuery.AnalysisType.class), - false, - null, - null // we don't care about merging strategy because merge is false - ); - - return queryLifecycleFactory - .factorize() - .runSimple(segmentMetadataQuery, escalator.createEscalatedAuthenticationResult(), Access.OK).getResults(); - } - - @VisibleForTesting - static RowSignature analysisToRowSignature(final SegmentAnalysis analysis) - { - final RowSignature.Builder rowSignatureBuilder = RowSignature.builder(); - for (Map.Entry entry : analysis.getColumns().entrySet()) { - if (entry.getValue().isError()) { - // Skip columns with analysis errors. - continue; - } - - ColumnType valueType = entry.getValue().getTypeSignature(); - - // this shouldn't happen, but if it does, first try to fall back to legacy type information field in case - // standard upgrade order was not followed for 0.22 to 0.23+, and if that also fails, then assume types are some - // flavor of COMPLEX. - if (valueType == null) { - // at some point in the future this can be simplified to the contents of the catch clause here, once the - // likelyhood of upgrading from some version lower than 0.23 is low - try { - valueType = ColumnType.fromString(entry.getValue().getType()); - if (valueType == null) { - valueType = ColumnType.ofComplex(entry.getValue().getType()); - } - } - catch (IllegalArgumentException ignored) { - valueType = ColumnType.UNKNOWN_COMPLEX; - } - } - - rowSignatureBuilder.add(entry.getKey(), valueType); - } - return ROW_SIGNATURE_INTERNER.intern(rowSignatureBuilder.build()); - } - - /** - * This method is not thread-safe and must be used only in unit tests. - */ - @VisibleForTesting - void setAvailableSegmentMetadata(final SegmentId segmentId, final AvailableSegmentMetadata availableSegmentMetadata) - { - final ConcurrentSkipListMap dataSourceSegments = segmentMetadataInfo - .computeIfAbsent( - segmentId.getDataSource(), - k -> new ConcurrentSkipListMap<>(SEGMENT_ORDER) - ); - if (dataSourceSegments.put(segmentId, availableSegmentMetadata) == null) { - totalSegments++; - } - } - - /** - * This is a helper method for unit tests to emulate heavy work done with {@link #lock}. - * It must be used only in unit tests. - */ - @VisibleForTesting - void doInLock(Runnable runnable) - { - synchronized (lock) { - runnable.run(); - } - } - - - /** - * ColumnTypeMergePolicy defines the rules of which type to use when faced with the possibility of different types - * for the same column from segment to segment. It is used to help compute a {@link RowSignature} for a table in - * Druid based on the segment metadata of all segments, merging the types of each column encountered to end up with - * a single type to represent it globally. - */ - @FunctionalInterface - public interface ColumnTypeMergePolicy - { - ColumnType merge(ColumnType existingType, ColumnType newType); - - @JsonCreator - static ColumnTypeMergePolicy fromString(String type) - { - if (LeastRestrictiveTypeMergePolicy.NAME.equalsIgnoreCase(type)) { - return LeastRestrictiveTypeMergePolicy.INSTANCE; - } - if (FirstTypeMergePolicy.NAME.equalsIgnoreCase(type)) { - return FirstTypeMergePolicy.INSTANCE; - } - throw new IAE("Unknown type [%s]", type); - } - } - - /** - * Classic logic, we use the first type we encounter. This policy is effectively 'newest first' because we iterated - * segments starting from the most recent time chunk, so this typically results in the most recently used type being - * chosen, at least for systems that are continuously updated with 'current' data. - * - * Since {@link ColumnTypeMergePolicy} are used to compute the SQL schema, at least in systems using SQL schemas which - * are partially or fully computed by this cache, this merge policy can result in query time errors if incompatible - * types are mixed if the chosen type is more restrictive than the types of some segments. If data is likely to vary - * in type across segments, consider using {@link LeastRestrictiveTypeMergePolicy} instead. - */ - public static class FirstTypeMergePolicy implements ColumnTypeMergePolicy - { - public static final String NAME = "latestInterval"; - private static final FirstTypeMergePolicy INSTANCE = new FirstTypeMergePolicy(); - - @Override - public ColumnType merge(ColumnType existingType, ColumnType newType) - { - if (existingType == null) { - return newType; - } - if (newType == null) { - return existingType; - } - // if any are json, are all json - if (ColumnType.NESTED_DATA.equals(newType) || ColumnType.NESTED_DATA.equals(existingType)) { - return ColumnType.NESTED_DATA; - } - // "existing type" is the 'newest' type, since we iterate the segments list by newest start time - return existingType; - } - - @Override - public int hashCode() - { - return Objects.hash(NAME); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - return o != null && getClass() == o.getClass(); - } - - @Override - public String toString() - { - return NAME; - } - } - - /** - * Resolves types using {@link ColumnType#leastRestrictiveType(ColumnType, ColumnType)} to find the ColumnType that - * can best represent all data contained across all segments. - */ - public static class LeastRestrictiveTypeMergePolicy implements ColumnTypeMergePolicy - { - public static final String NAME = "leastRestrictive"; - private static final LeastRestrictiveTypeMergePolicy INSTANCE = new LeastRestrictiveTypeMergePolicy(); - - @Override - public ColumnType merge(ColumnType existingType, ColumnType newType) - { - try { - return ColumnType.leastRestrictiveType(existingType, newType); - } - catch (Types.IncompatibleTypeException incompatibleTypeException) { - // fall back to first encountered type if they are not compatible for some reason - return FirstTypeMergePolicy.INSTANCE.merge(existingType, newType); - } - } - - @Override - public int hashCode() - { - return Objects.hash(NAME); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - return o != null && getClass() == o.getClass(); - } - - @Override - public String toString() - { - return NAME; - } - } } diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java index de11d2b491e3..5857619e3947 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfig.java @@ -37,8 +37,8 @@ public class SegmentMetadataCacheConfig private Period metadataRefreshPeriod = new Period("PT1M"); @JsonProperty - private SegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = - new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); + private AbstractSegmentMetadataCache.ColumnTypeMergePolicy metadataColumnTypeMergePolicy = + new AbstractSegmentMetadataCache.LeastRestrictiveTypeMergePolicy(); public static SegmentMetadataCacheConfig create() { @@ -65,7 +65,7 @@ public boolean isAwaitInitializationOnStart() return awaitInitializationOnStart; } - public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() + public AbstractSegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() { return metadataColumnTypeMergePolicy; } diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index 7fe2cd76e129..c79962c0d111 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -75,7 +75,7 @@ public class MetadataResource private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; private final DruidCoordinator coordinator; - private final @Nullable SegmentMetadataCache segmentMetadataCache; + private final @Nullable SegmentMetadataCache segmentMetadataCache; @Inject public MetadataResource( @@ -83,7 +83,7 @@ public MetadataResource( IndexerMetadataStorageCoordinator metadataStorageCoordinator, AuthorizerMapper authorizerMapper, DruidCoordinator coordinator, - @Nullable SegmentMetadataCache segmentMetadataCache + @Nullable SegmentMetadataCache segmentMetadataCache ) { this.segmentsMetadataManager = segmentsMetadataManager; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index 6a333a9dde02..1c9256da0fc8 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -47,6 +47,7 @@ import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.QueryableIndex; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; @@ -89,7 +90,7 @@ public class SegmentDataCacheConcurrencyTest extends SegmentMetadataCacheCommon private File tmpDir; private TestServerInventoryView inventoryView; private BrokerServerView serverView; - private SegmentMetadataCache schema; + private AbstractSegmentMetadataCache schema; private ExecutorService exec; @Before @@ -115,10 +116,10 @@ public void tearDown() throws Exception } /** - * This tests the contention between three components, {@link SegmentMetadataCache}, + * This tests the contention between three components, {@link AbstractSegmentMetadataCache}, * {@code InventoryView}, and {@link BrokerServerView}. It first triggers * refreshing {@code SegmentMetadataCache}. To mimic some heavy work done with - * {@link SegmentMetadataCache#lock}, {@link SegmentMetadataCache#buildDruidTable} + * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDruidTable} * is overridden to sleep before doing real work. While refreshing * {@code SegmentMetadataCache}, more new segments are added to * {@code InventoryView}, which triggers updates of {@code BrokerServerView}. @@ -140,7 +141,7 @@ public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerV ) { @Override - public DataSourceInformation buildDruidTable(final String dataSource) + public RowSignature buildDruidTable(final String dataSource) { doInLock(() -> { try { @@ -224,11 +225,11 @@ public CallbackAction serverSegmentRemoved(DruidServerMetadata server, DataSegme } /** - * This tests the contention between two methods of {@link SegmentMetadataCache}: - * {@link SegmentMetadataCache#refresh} and - * {@link SegmentMetadataCache#getSegmentMetadataSnapshot()}. It first triggers + * This tests the contention between two methods of {@link AbstractSegmentMetadataCache}: + * {@link AbstractSegmentMetadataCache#refresh} and + * {@link AbstractSegmentMetadataCache#getSegmentMetadataSnapshot()}. It first triggers * refreshing {@code SegmentMetadataCache}. To mimic some heavy work done with - * {@link SegmentMetadataCache#lock}, {@link SegmentMetadataCache#buildDruidTable} + * {@link AbstractSegmentMetadataCache#lock}, {@link AbstractSegmentMetadataCache#buildDruidTable} * is overridden to sleep before doing real work. While refreshing * {@code SegmentMetadataCache}, {@code getSegmentMetadataSnapshot()} is continuously * called to mimic reading the segments table of SystemSchema. All these calls @@ -248,7 +249,7 @@ public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() ) { @Override - public DataSourceInformation buildDruidTable(final String dataSource) + public RowSignature buildDruidTable(final String dataSource) { doInLock(() -> { try { diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java index 5717ab476c6f..42f365b9af85 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheCommon.java @@ -346,7 +346,7 @@ QueryLifecycleFactory getQueryLifecycleFactory(QuerySegmentWalker walker) } public void checkRefreshShouldEmitMetrics( - SegmentMetadataCache schema, + AbstractSegmentMetadataCache schema, String dataSource, StubServiceEmitter emitter, CountDownLatch addSegmentLatch @@ -366,7 +366,7 @@ public void checkRefreshShouldEmitMetrics( emitter.verifyEmitted("metadatacache/refresh/count", ImmutableMap.of(DruidMetrics.DATASOURCE, dataSource), 1); } - public void checkNullAvailableSegmentMetadata(SegmentMetadataCache schema) throws IOException + public void checkNullAvailableSegmentMetadata(AbstractSegmentMetadataCache schema) throws IOException { final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); final List segments = segmentMetadatas.values() @@ -388,7 +388,7 @@ public void checkNullAvailableSegmentMetadata(SegmentMetadataCache schema) throw Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); } - public void checkNullDatasource(SegmentMetadataCache schema) throws IOException + public void checkNullDatasource(AbstractSegmentMetadataCache schema) throws IOException { final Map segmentMetadatas = schema.getSegmentMetadataSnapshot(); final List segments = segmentMetadatas.values() @@ -411,7 +411,7 @@ public void checkNullDatasource(SegmentMetadataCache schema) throws IOException Assert.assertEquals(5, schema.getSegmentMetadataSnapshot().size()); } - public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) + public void checkAvailableSegmentMetadataNumRows(AbstractSegmentMetadataCache schema) { Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); final List segments = segmentsMetadata.values() @@ -459,7 +459,7 @@ public void checkAvailableSegmentMetadataNumRows(SegmentMetadataCache schema) Assert.assertEquals(updatedMetadata.getNumReplicas(), currentMetadata.getNumReplicas()); } - public void checkRunSegmentMetadataQueryWithContext(SegmentMetadataCache schema, QueryLifecycleFactory factoryMock, QueryLifecycle lifecycleMock) + public void checkRunSegmentMetadataQueryWithContext(AbstractSegmentMetadataCache schema, QueryLifecycleFactory factoryMock, QueryLifecycle lifecycleMock) { Map queryContext = ImmutableMap.of( QueryContexts.PRIORITY_KEY, 5, diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java index d625229972fe..119191dc767f 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java @@ -51,7 +51,7 @@ public void testDefaultConfig() final SegmentMetadataCacheConfig config = provider.get(); Assert.assertFalse(config.isAwaitInitializationOnStart()); Assert.assertEquals(Period.minutes(1), config.getMetadataRefreshPeriod()); - Assert.assertEquals(new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); + Assert.assertEquals(new AbstractSegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); } @Test @@ -74,7 +74,7 @@ public void testCustomizedConfig() Assert.assertFalse(config.isAwaitInitializationOnStart()); Assert.assertEquals(Period.minutes(2), config.getMetadataRefreshPeriod()); Assert.assertEquals( - new SegmentMetadataCache.FirstTypeMergePolicy(), + new AbstractSegmentMetadataCache.FirstTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy() ); } diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java index 78ba2add3627..503431341abf 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java @@ -63,7 +63,7 @@ public class SegmentMetadataCacheTest extends SegmentMetadataCacheCommon // Timeout to allow (rapid) debugging, while not blocking tests with errors. private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); - private SegmentMetadataCache runningSchema; + private SegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); @@ -85,15 +85,15 @@ public void tearDown() throws Exception walker.close(); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException + public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException + public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException { Preconditions.checkState(runningSchema == null); - runningSchema = new SegmentMetadataCache( + runningSchema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, config, @@ -103,9 +103,9 @@ public SegmentMetadataCache buildSchemaMarkAndTableLatch( ) { @Override - public DataSourceInformation buildDruidTable(String dataSource) + public RowSignature buildDruidTable(String dataSource) { - DataSourceInformation table = super.buildDruidTable(dataSource); + RowSignature table = super.buildDruidTable(dataSource); buildTableLatch.countDown(); return table; } @@ -126,7 +126,7 @@ public void markDataSourceAsNeedRebuild(String datasource) @Test public void testGetTableMap() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); final Set tableNames = schema.getDatasourceNames(); @@ -136,7 +136,7 @@ public void testGetTableMap() throws InterruptedException @Test public void testGetTableMapFoo() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -164,7 +164,7 @@ public void testGetTableMapFoo() throws InterruptedException @Test public void testGetTableMapFoo2() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo2"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -185,12 +185,12 @@ public void testGetTableMapSomeTable() throws InterruptedException { // using 'newest first' column type merge strategy, the types are expected to be the types defined in the newer // segment, except for json, which is special handled - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch( + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch( new SegmentMetadataCacheConfig() { @Override - public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() + public AbstractSegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() { - return new SegmentMetadataCache.FirstTypeMergePolicy(); + return new AbstractSegmentMetadataCache.FirstTypeMergePolicy(); } } ); @@ -232,7 +232,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt { // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); final RowSignature fooRowSignature = fooDs.getRowSignature(); @@ -269,35 +269,35 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt /** * This tests that {@link AvailableSegmentMetadata#getNumRows()} is correct in case - * of multiple replicas i.e. when {@link SegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} + * of multiple replicas i.e. when {@link AbstractSegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} * is called more than once for same segment * @throws InterruptedException */ @Test public void testAvailableSegmentMetadataNumRows() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkAvailableSegmentMetadataNumRows(schema); } @Test public void testNullDatasource() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullDatasource(schema); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullAvailableSegmentMetadata(schema); } @Test public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); final List segments = segmentsMetadata.values() .stream() @@ -354,7 +354,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -395,7 +395,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -440,7 +440,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -482,7 +482,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -521,7 +521,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -577,7 +577,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -636,7 +636,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr { String datasource = "serverSegmentRemoveTest"; CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -669,7 +669,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -715,7 +715,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -785,7 +785,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); // Need to create schema for this test because the available schemas don't mock the QueryLifecycleFactory, which I need for this test. - SegmentMetadataCache mySchema = new SegmentMetadataCache( + SegmentMetadataCache mySchema = new SegmentMetadataCache( factoryMock, serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -817,7 +817,7 @@ public void testSegmentMetadataColumnType() new ColumnAnalysis(ColumnType.DOUBLE, ColumnType.DOUBLE.asTypeString(), false, true, 1234, 26, null, null, null) ); - RowSignature signature = SegmentMetadataCache.analysisToRowSignature( + RowSignature signature = AbstractSegmentMetadataCache.analysisToRowSignature( new SegmentAnalysis( "id", ImmutableList.of(Intervals.utc(1L, 2L)), @@ -844,7 +844,7 @@ public void testSegmentMetadataColumnType() @Test public void testSegmentMetadataFallbackType() { - RowSignature signature = SegmentMetadataCache.analysisToRowSignature( + RowSignature signature = AbstractSegmentMetadataCache.analysisToRowSignature( new SegmentAnalysis( "id", ImmutableList.of(Intervals.utc(1L, 2L)), @@ -905,7 +905,7 @@ public void testSegmentMetadataFallbackType() @Test public void testStaleDatasourceRefresh() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Set segments = new HashSet<>(); Set datasources = new HashSet<>(); datasources.add("wat"); @@ -920,7 +920,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept String dataSource = "xyz"; CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); - SegmentMetadataCache schema = new SegmentMetadataCache( + SegmentMetadataCache schema = new SegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index b111c1147cc8..200c3e71ce7c 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -74,7 +74,6 @@ import org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory; import org.apache.druid.java.util.common.lifecycle.Lifecycle; import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.metadata.MetadataRuleManager; import org.apache.druid.metadata.MetadataRuleManagerConfig; @@ -88,11 +87,10 @@ import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.lookup.LookupSerdeModule; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; -import org.apache.druid.segment.metadata.DataSourceInformation; +import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.ClientQuerySegmentWalker; -import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.audit.AuditManagerProvider; import org.apache.druid.server.coordinator.CoordinatorConfigManager; import org.apache.druid.server.coordinator.DruidCoordinator; @@ -128,7 +126,6 @@ import org.apache.druid.server.lookup.cache.LookupCoordinatorManagerConfig; import org.apache.druid.server.metrics.ServiceStatusMonitor; import org.apache.druid.server.router.TieredBrokerConfig; -import org.apache.druid.server.security.Escalator; import org.eclipse.jetty.server.Server; import org.joda.time.Duration; @@ -228,7 +225,7 @@ public void configure(Binder binder) } else { binder.bind(CoordinatorTimeline.class).to(CoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, CoordinatorServerView.class); - binder.bind(SegmentMetadataCache.class).toProvider(Providers.of(null)); + binder.bind(AbstractSegmentMetadataCache.class).toProvider(Providers.of(null)); } binder.bind(SegmentsMetadataManager.class) @@ -484,48 +481,7 @@ public void configure(Binder binder) binder.bind(CoordinatorTimeline.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); binder.bind(TimelineServerView.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, QueryableCoordinatorServerView.class); - binder.bind(new TypeLiteral>() {}) - .toProvider(SegmentMetadataCacheProvider.class).in(ManageLifecycle.class); - } - - private static class SegmentMetadataCacheProvider implements Provider> - { - private final QueryLifecycleFactory queryLifecycleFactory; - private final TimelineServerView serverView; - private final SegmentMetadataCacheConfig config; - private final Escalator escalator; - private final InternalQueryConfig internalQueryConfig; - private final ServiceEmitter emitter; - - @Inject - public SegmentMetadataCacheProvider( - QueryLifecycleFactory queryLifecycleFactory, - TimelineServerView serverView, - SegmentMetadataCacheConfig config, - Escalator escalator, - InternalQueryConfig internalQueryConfig, - ServiceEmitter emitter - ) - { - this.queryLifecycleFactory = queryLifecycleFactory; - this.serverView = serverView; - this.config = config; - this.escalator = escalator; - this.internalQueryConfig = internalQueryConfig; - this.emitter = emitter; - } - - @Override - public SegmentMetadataCache get() - { - return new SegmentMetadataCache<>( - queryLifecycleFactory, - serverView, - config, - escalator, - internalQueryConfig, - emitter); - } + LifecycleModule.register(binder, SegmentMetadataCache.class); } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 14803ea31417..e761ecd702d6 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -29,8 +29,9 @@ import org.apache.druid.guice.ManageLifecycle; import org.apache.druid.java.util.emitter.EmittingLogger; import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.segment.metadata.DataSourceInformation; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; @@ -48,27 +49,19 @@ * metadata about a dataSource which is blended with catalog "logical" metadata * to provide the final user-view of each dataSource. *

    - * This class extends {@link SegmentMetadataCache} and introduces following changes, + * This class extends {@link AbstractSegmentMetadataCache} and introduces following changes, *

      *
    • The refresh mechanism now includes polling the coordinator for dataSource schema, * and falling back to running {@link org.apache.druid.query.metadata.metadata.SegmentMetadataQuery}.
      • - *
    • It builds and caches {@link PhysicalDatasourceMetadata} object as table - * schema instead of {@link DataSourceInformation}.
      • + *
    • It builds and caches {@link PhysicalDatasourceMetadata} object for the table schema
      • *
    */ @ManageLifecycle -public class BrokerSegmentMetadataCache extends SegmentMetadataCache +public class BrokerSegmentMetadataCache extends AbstractSegmentMetadataCache { private static final EmittingLogger log = new EmittingLogger(BrokerSegmentMetadataCache.class); private final PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; - - /** - * Manages tables of PhysicalDataSourceMetadata. This manager is used to retrieve and store - * information related to dataSources. - * This structure can be accessed by {@link #cacheExec} and {@link #callbackExec} threads. - */ - //private final TableManager<> tableManager = new TableManager<>(); private final CoordinatorClient coordinatorClient; @Inject @@ -97,8 +90,10 @@ public BrokerSegmentMetadataCache( /** * Refreshes the set of segments in two steps: - * 1. Polls the coordinator for the dataSource schema to update the {@code tables}. - * 2. Refreshes the remaining set of segments by executing a SegmentMetadataQuery. + *
      + *
    • Polls the coordinator for the dataSource schema to update the {@code tables}.
      • + *
    • Refreshes the remaining set of segments by executing a SegmentMetadataQuery.
      • + *
    */ @Override public void refresh(final Set segmentsToRefresh, final Set dataSourcesToRebuild) throws IOException @@ -112,13 +107,16 @@ public void refresh(final Set segmentsToRefresh, final Set da // Fetch dataSource information from the Coordinator try { FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) - .forEach(item -> polledDataSourceMetadata.put( - item.getDataSource(), - physicalDatasourceMetadataBuilder.build(item) + .forEach(dataSourceInformation -> polledDataSourceMetadata.put( + dataSourceInformation.getDataSource(), + physicalDatasourceMetadataBuilder.build( + dataSourceInformation.getDataSource(), + dataSourceInformation.getRowSignature() + ) )); } catch (Exception e) { - log.warn(e, "Exception querying coordinator to fetch dataSourceInformation."); + log.warn("Failed to query dataSource information from the Coordinator."); } // remove any extra dataSources returned @@ -147,16 +145,17 @@ public void refresh(final Set segmentsToRefresh, final Set da // Rebuild the dataSources. for (String dataSource : dataSourcesToRebuild) { - final DataSourceInformation druidTable = buildDruidTable(dataSource); - if (druidTable == null) { + final RowSignature rowSignature = buildDruidTable(dataSource); + if (rowSignature == null) { log.info("dataSource [%s] no longer exists, all metadata removed.", dataSource); tables.remove(dataSource); return; } - final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(druidTable); + + final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(dataSource, rowSignature); final PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); if (oldTable == null || !oldTable.getRowSignature().equals(physicalDatasourceMetadata.getRowSignature())) { - log.info("[%s] has new signature: %s.", dataSource, druidTable.getRowSignature()); + log.info("[%s] has new signature: %s.", dataSource, rowSignature); } else { log.debug("[%s] signature is unchanged.", dataSource); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index c5922b066885..4ef9d63788e9 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -22,6 +22,7 @@ import com.google.inject.Inject; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.TableDataSource; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.SegmentManager; @@ -46,9 +47,8 @@ public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, Segmen /** * Builds physical metadata for the given data source. */ - PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) + PhysicalDatasourceMetadata build(final String dataSource, final RowSignature rowSignature) { - final String dataSource = dataSourceInformation.getDataSource(); final TableDataSource tableDataSource; // to be a GlobalTableDataSource instead of a TableDataSource, it must appear on all servers (inferred by existing @@ -66,7 +66,7 @@ PhysicalDatasourceMetadata build(DataSourceInformation dataSourceInformation) } return new PhysicalDatasourceMetadata( tableDataSource, - dataSourceInformation.getRowSignature(), + rowSignature, isJoinable, isBroadcast ); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java index 25e6096bb190..a2f5ef4b5395 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheConfigTest.java @@ -24,7 +24,7 @@ import org.apache.druid.guice.GuiceInjectors; import org.apache.druid.guice.JsonConfigProvider; import org.apache.druid.guice.JsonConfigurator; -import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.sql.calcite.planner.CalcitePlannerModule; import org.joda.time.Period; import org.junit.Assert; @@ -52,7 +52,7 @@ public void testDefaultConfig() Assert.assertTrue(config.isAwaitInitializationOnStart()); Assert.assertTrue(config.isMetadataSegmentCacheEnable()); Assert.assertEquals(Period.minutes(1), config.getMetadataRefreshPeriod()); - Assert.assertEquals(new SegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); + Assert.assertEquals(new AbstractSegmentMetadataCache.LeastRestrictiveTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy()); } @Test @@ -76,7 +76,7 @@ public void testCustomizedConfig() Assert.assertTrue(config.isMetadataSegmentCacheEnable()); Assert.assertEquals(Period.minutes(2), config.getMetadataRefreshPeriod()); Assert.assertEquals( - new SegmentMetadataCache.FirstTypeMergePolicy(), + new AbstractSegmentMetadataCache.FirstTypeMergePolicy(), config.getMetadataColumnTypeMergePolicy() ); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index f4b55fb5929a..a913dfcc9a5b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -52,9 +52,9 @@ import org.apache.druid.segment.join.Joinable; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; +import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; import org.apache.druid.segment.metadata.DataSourceInformation; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheCommon; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.QueryLifecycleFactory; @@ -105,7 +105,6 @@ public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon Set segmentDataSourceNames; Set joinableDataSourceNames; JoinableFactory globalTableJoinable; - CountDownLatch getDatasourcesLatch = new CountDownLatch(1); private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); @@ -122,7 +121,6 @@ public void setUp() throws Exception @Override public Set getDataSourceNames() { - getDatasourcesLatch.countDown(); return segmentDataSourceNames; } }; @@ -178,9 +176,9 @@ public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMeta ) { @Override - public DataSourceInformation buildDruidTable(String dataSource) + public RowSignature buildDruidTable(String dataSource) { - DataSourceInformation table = super.buildDruidTable(dataSource); + RowSignature table = super.buildDruidTable(dataSource); buildTableLatch.countDown(); return table; } @@ -410,9 +408,9 @@ public void testGetTableMapSomeTable() throws InterruptedException BrokerSegmentMetadataCache schema = buildSchemaMarkAndTableLatch( new BrokerSegmentMetadataCacheConfig() { @Override - public SegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() + public AbstractSegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() { - return new SegmentMetadataCache.FirstTypeMergePolicy(); + return new AbstractSegmentMetadataCache.FirstTypeMergePolicy(); } }, new NoopCoordinatorClient() @@ -500,7 +498,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt /** * This tests that {@link AvailableSegmentMetadata#getNumRows()} is correct in case - * of multiple replicas i.e. when {@link SegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} + * of multiple replicas i.e. when {@link AbstractSegmentMetadataCache#addSegment(DruidServerMetadata, DataSegment)} * is called more than once for same segment * @throws InterruptedException */ diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java index ccdbef25461b..3847bddd43ac 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java @@ -28,7 +28,6 @@ import org.apache.druid.segment.join.Joinable; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.loading.SegmentLoader; -import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.SegmentManager; import org.apache.druid.sql.calcite.table.DatasourceTable; import org.easymock.EasyMock; @@ -38,12 +37,9 @@ import java.util.Optional; import java.util.Set; -import java.util.concurrent.CountDownLatch; public class PhysicalDataSourceMetadataBuilderTest { - private CountDownLatch getDatasourcesLatch = new CountDownLatch(1); - private Set segmentDataSourceNames; private Set joinableDataSourceNames; private SegmentManager segmentManager; @@ -61,7 +57,6 @@ public void setUp() @Override public Set getDataSourceNames() { - getDatasourcesLatch.countDown(); return segmentDataSourceNames; } }; @@ -94,34 +89,28 @@ public void testBuild() segmentDataSourceNames.add("foo"); joinableDataSourceNames.add("foo"); - DataSourceInformation foo = - new DataSourceInformation( - "foo", - RowSignature.builder() - .add("c1", ColumnType.FLOAT) - .add("c2", ColumnType.DOUBLE) - .build() - ); + RowSignature fooSignature = + RowSignature.builder() + .add("c1", ColumnType.FLOAT) + .add("c2", ColumnType.DOUBLE) + .build(); - DataSourceInformation bar = - new DataSourceInformation( - "bar", - RowSignature.builder() - .add("d1", ColumnType.FLOAT) - .add("d2", ColumnType.DOUBLE) - .build() - ); + RowSignature barSignature = + RowSignature.builder() + .add("d1", ColumnType.FLOAT) + .add("d2", ColumnType.DOUBLE) + .build(); - DatasourceTable.PhysicalDatasourceMetadata fooDs = physicalDatasourceMetadataBuilder.build(foo); + DatasourceTable.PhysicalDatasourceMetadata fooDs = physicalDatasourceMetadataBuilder.build("foo", fooSignature); Assert.assertTrue(fooDs.isJoinable()); Assert.assertTrue(fooDs.isBroadcast()); - Assert.assertEquals(fooDs.dataSource().getName(), foo.getDataSource()); - Assert.assertEquals(fooDs.getRowSignature(), foo.getRowSignature()); + Assert.assertEquals(fooDs.dataSource().getName(), "foo"); + Assert.assertEquals(fooDs.getRowSignature(), fooSignature); - DatasourceTable.PhysicalDatasourceMetadata barDs = physicalDatasourceMetadataBuilder.build(bar); + DatasourceTable.PhysicalDatasourceMetadata barDs = physicalDatasourceMetadataBuilder.build("bar", barSignature); Assert.assertFalse(barDs.isJoinable()); Assert.assertFalse(barDs.isBroadcast()); - Assert.assertEquals(barDs.dataSource().getName(), bar.getDataSource()); - Assert.assertEquals(barDs.getRowSignature(), bar.getRowSignature()); + Assert.assertEquals(barDs.dataSource().getName(), "bar"); + Assert.assertEquals(barDs.getRowSignature(), barSignature); } } From 7badce1612f48ffbf9e41f38417b1224d0a6323b Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 00:19:58 +0530 Subject: [PATCH 34/36] Rename SegmentMetadataCache to CoordinatorSegmentMetadataCache --- ...ruidSchemaInternRowSignatureBenchmark.java | 4 +- ...a => CoordinatorSegmentMetadataCache.java} | 4 +- .../druid/server/http/MetadataResource.java | 20 +++---- ...inatorSegmentMetadataCacheConfigTest.java} | 2 +- ... CoordinatorSegmentMetadataCacheTest.java} | 54 +++++++++---------- .../SegmentDataCacheConcurrencyTest.java | 4 +- .../server/http/MetadataResourceTest.java | 18 +++---- .../org/apache/druid/cli/CliCoordinator.java | 4 +- .../schema/BrokerSegmentMetadataCache.java | 1 - .../PhysicalDatasourceMetadataBuilder.java | 1 - .../BrokerSegmentMetadataCacheTest.java | 6 +-- 11 files changed, 57 insertions(+), 61 deletions(-) rename server/src/main/java/org/apache/druid/segment/metadata/{SegmentMetadataCache.java => CoordinatorSegmentMetadataCache.java} (96%) rename server/src/test/java/org/apache/druid/segment/metadata/{SegmentMetadataCacheConfigTest.java => CoordinatorSegmentMetadataCacheConfigTest.java} (98%) rename server/src/test/java/org/apache/druid/segment/metadata/{SegmentMetadataCacheTest.java => CoordinatorSegmentMetadataCacheTest.java} (94%) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java index 5f0634f283a4..85a9931ab037 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DruidSchemaInternRowSignatureBenchmark.java @@ -30,7 +30,7 @@ import org.apache.druid.query.metadata.metadata.ColumnAnalysis; import org.apache.druid.query.metadata.metadata.SegmentAnalysis; import org.apache.druid.segment.column.ColumnType; -import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.coordination.DruidServerMetadata; @@ -68,7 +68,7 @@ public class DruidSchemaInternRowSignatureBenchmark { private SegmentMetadataCacheForBenchmark cache; - private static class SegmentMetadataCacheForBenchmark extends SegmentMetadataCache + private static class SegmentMetadataCacheForBenchmark extends CoordinatorSegmentMetadataCache { public SegmentMetadataCacheForBenchmark( final QueryLifecycleFactory queryLifecycleFactory, diff --git a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java similarity index 96% rename from server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java rename to server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index 83c67532d6e2..a24308ff4723 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/SegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -39,12 +39,12 @@ * datasources. The cache provides metadata about a dataSource, see {@link DataSourceInformation}. */ @ManageLifecycle -public class SegmentMetadataCache extends AbstractSegmentMetadataCache +public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCache { private static final EmittingLogger log = new EmittingLogger(AbstractSegmentMetadataCache.class); @Inject - public SegmentMetadataCache( + public CoordinatorSegmentMetadataCache( QueryLifecycleFactory queryLifecycleFactory, TimelineServerView serverView, SegmentMetadataCacheConfig config, diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java index c79962c0d111..9a8fb176acfa 100644 --- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java +++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java @@ -30,8 +30,8 @@ import org.apache.druid.indexing.overlord.Segments; import org.apache.druid.metadata.SegmentsMetadataManager; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; import org.apache.druid.segment.metadata.DataSourceInformation; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.JettyUtils; import org.apache.druid.server.coordinator.DruidCoordinator; import org.apache.druid.server.http.security.DatasourceResourceFilter; @@ -75,7 +75,7 @@ public class MetadataResource private final IndexerMetadataStorageCoordinator metadataStorageCoordinator; private final AuthorizerMapper authorizerMapper; private final DruidCoordinator coordinator; - private final @Nullable SegmentMetadataCache segmentMetadataCache; + private final @Nullable CoordinatorSegmentMetadataCache coordinatorSegmentMetadataCache; @Inject public MetadataResource( @@ -83,14 +83,14 @@ public MetadataResource( IndexerMetadataStorageCoordinator metadataStorageCoordinator, AuthorizerMapper authorizerMapper, DruidCoordinator coordinator, - @Nullable SegmentMetadataCache segmentMetadataCache + @Nullable CoordinatorSegmentMetadataCache coordinatorSegmentMetadataCache ) { this.segmentsMetadataManager = segmentsMetadataManager; this.metadataStorageCoordinator = metadataStorageCoordinator; this.authorizerMapper = authorizerMapper; this.coordinator = coordinator; - this.segmentMetadataCache = segmentMetadataCache; + this.coordinatorSegmentMetadataCache = coordinatorSegmentMetadataCache; } @GET @@ -203,8 +203,8 @@ private Response getAllUsedSegmentsWithAdditionalDetails( : coordinator.getReplicationFactor(segment.getId()); Long numRows = null; - if (null != segmentMetadataCache) { - AvailableSegmentMetadata availableSegmentMetadata = segmentMetadataCache.getAvailableSegmentMetadata( + if (null != coordinatorSegmentMetadataCache) { + AvailableSegmentMetadata availableSegmentMetadata = coordinatorSegmentMetadataCache.getAvailableSegmentMetadata( segment.getDataSource(), segment.getId() ); @@ -226,8 +226,8 @@ private Response getAllUsedSegmentsWithAdditionalDetails( Stream finalSegments = segmentStatus; // conditionally add realtime segments information - if (null != includeRealtimeSegments && null != segmentMetadataCache) { - final Stream realtimeSegmentStatus = segmentMetadataCache + if (null != includeRealtimeSegments && null != coordinatorSegmentMetadataCache) { + final Stream realtimeSegmentStatus = coordinatorSegmentMetadataCache .getSegmentMetadataSnapshot() .values() .stream() @@ -366,10 +366,10 @@ public Response getDataSourceInformation( List dataSources ) { - if (null == segmentMetadataCache) { + if (null == coordinatorSegmentMetadataCache) { return Response.status(Response.Status.NOT_FOUND).build(); } - Map dataSourceSchemaMap = segmentMetadataCache.getDataSourceInformationMap(); + Map dataSourceSchemaMap = coordinatorSegmentMetadataCache.getDataSourceInformationMap(); List results = new ArrayList<>(); List dataSourcesToRetain = (null == dataSources) ? new ArrayList<>(dataSourceSchemaMap.keySet()) : dataSources; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheConfigTest.java similarity index 98% rename from server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java rename to server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheConfigTest.java index 119191dc767f..9668466396e0 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheConfigTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheConfigTest.java @@ -33,7 +33,7 @@ /** * Pathetic little unit test just to keep Jacoco happy. */ -public class SegmentMetadataCacheConfigTest +public class CoordinatorSegmentMetadataCacheConfigTest { private static final String CONFIG_BASE = "druid.coordinator.segmentMetadataCache"; diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java similarity index 94% rename from server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java rename to server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java index 503431341abf..90664e7c87d0 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java @@ -58,12 +58,12 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -public class SegmentMetadataCacheTest extends SegmentMetadataCacheCommon +public class CoordinatorSegmentMetadataCacheTest extends SegmentMetadataCacheCommon { // Timeout to allow (rapid) debugging, while not blocking tests with errors. private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); - static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); - private SegmentMetadataCache runningSchema; + private static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S"); + private CoordinatorSegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); @@ -85,15 +85,15 @@ public void tearDown() throws Exception walker.close(); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException + public CoordinatorSegmentMetadataCache buildSchemaMarkAndTableLatch() throws InterruptedException { return buildSchemaMarkAndTableLatch(SEGMENT_CACHE_CONFIG_DEFAULT); } - public SegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException + public CoordinatorSegmentMetadataCache buildSchemaMarkAndTableLatch(SegmentMetadataCacheConfig config) throws InterruptedException { Preconditions.checkState(runningSchema == null); - runningSchema = new SegmentMetadataCache( + runningSchema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, config, @@ -126,7 +126,7 @@ public void markDataSourceAsNeedRebuild(String datasource) @Test public void testGetTableMap() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Assert.assertEquals(ImmutableSet.of(DATASOURCE1, DATASOURCE2, SOME_DATASOURCE), schema.getDatasourceNames()); final Set tableNames = schema.getDatasourceNames(); @@ -136,7 +136,7 @@ public void testGetTableMap() throws InterruptedException @Test public void testGetTableMapFoo() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -164,7 +164,7 @@ public void testGetTableMapFoo() throws InterruptedException @Test public void testGetTableMapFoo2() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource("foo2"); final RowSignature fooRowSignature = fooDs.getRowSignature(); List columnNames = fooRowSignature.getColumnNames(); @@ -185,7 +185,7 @@ public void testGetTableMapSomeTable() throws InterruptedException { // using 'newest first' column type merge strategy, the types are expected to be the types defined in the newer // segment, except for json, which is special handled - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch( + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch( new SegmentMetadataCacheConfig() { @Override public AbstractSegmentMetadataCache.ColumnTypeMergePolicy getMetadataColumnTypeMergePolicy() @@ -232,7 +232,7 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt { // using 'least restrictive' column type merge strategy, the types are expected to be the types defined as the // least restrictive blend across all segments - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); final DataSourceInformation fooDs = schema.getDatasource(SOME_DATASOURCE); final RowSignature fooRowSignature = fooDs.getRowSignature(); @@ -276,28 +276,28 @@ public void testGetTableMapSomeTableLeastRestrictiveTypeMerge() throws Interrupt @Test public void testAvailableSegmentMetadataNumRows() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkAvailableSegmentMetadataNumRows(schema); } @Test public void testNullDatasource() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullDatasource(schema); } @Test public void testNullAvailableSegmentMetadata() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); checkNullAvailableSegmentMetadata(schema); } @Test public void testAvailableSegmentMetadataIsRealtime() throws InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Map segmentsMetadata = schema.getSegmentMetadataSnapshot(); final List segments = segmentsMetadata.values() .stream() @@ -354,7 +354,7 @@ public void testSegmentAddedCallbackAddNewHistoricalSegment() throws Interrupted { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -395,7 +395,7 @@ public void testSegmentAddedCallbackAddExistingSegment() throws InterruptedExcep { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -440,7 +440,7 @@ public void testSegmentAddedCallbackAddNewRealtimeSegment() throws InterruptedEx { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -482,7 +482,7 @@ public void testSegmentAddedCallbackAddNewBroadcastSegment() throws InterruptedE { String datasource = "newSegmentAddTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -521,7 +521,7 @@ public void testSegmentRemovedCallbackEmptyDataSourceAfterRemove() throws Interr String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -577,7 +577,7 @@ public void testSegmentRemovedCallbackNonEmptyDataSourceAfterRemove() throws Int String datasource = "segmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(2); CountDownLatch removeSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -636,7 +636,7 @@ public void testServerSegmentRemovedCallbackRemoveUnknownSegment() throws Interr { String datasource = "serverSegmentRemoveTest"; CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -669,7 +669,7 @@ public void testServerSegmentRemovedCallbackRemoveBrokerSegment() throws Interru String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -715,7 +715,7 @@ public void testServerSegmentRemovedCallbackRemoveHistoricalSegment() throws Int String datasource = "serverSegmentRemoveTest"; CountDownLatch addSegmentLatch = new CountDownLatch(1); CountDownLatch removeServerSegmentLatch = new CountDownLatch(1); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -785,7 +785,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception QueryLifecycle lifecycleMock = EasyMock.createMock(QueryLifecycle.class); // Need to create schema for this test because the available schemas don't mock the QueryLifecycleFactory, which I need for this test. - SegmentMetadataCache mySchema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache mySchema = new CoordinatorSegmentMetadataCache( factoryMock, serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -905,7 +905,7 @@ public void testSegmentMetadataFallbackType() @Test public void testStaleDatasourceRefresh() throws IOException, InterruptedException { - SegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); + CoordinatorSegmentMetadataCache schema = buildSchemaMarkAndTableLatch(); Set segments = new HashSet<>(); Set datasources = new HashSet<>(); datasources.add("wat"); @@ -920,7 +920,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept String dataSource = "xyz"; CountDownLatch addSegmentLatch = new CountDownLatch(2); StubServiceEmitter emitter = new StubServiceEmitter("broker", "host"); - SegmentMetadataCache schema = new SegmentMetadataCache( + CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java index 1c9256da0fc8..a26b82722f86 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentDataCacheConcurrencyTest.java @@ -131,7 +131,7 @@ public void tearDown() throws Exception public void testSegmentMetadataRefreshAndInventoryViewAddSegmentAndBrokerServerViewGetTimeline() throws InterruptedException, ExecutionException, TimeoutException { - schema = new SegmentMetadataCache( + schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, @@ -239,7 +239,7 @@ public CallbackAction serverSegmentRemoved(DruidServerMetadata server, DataSegme public void testSegmentMetadataRefreshAndDruidSchemaGetSegmentMetadata() throws InterruptedException, ExecutionException, TimeoutException { - schema = new SegmentMetadataCache( + schema = new CoordinatorSegmentMetadataCache( getQueryLifecycleFactory(walker), serverView, SEGMENT_CACHE_CONFIG_DEFAULT, diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java index e122334946e5..a430081c10a7 100644 --- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java +++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java @@ -32,8 +32,8 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.metadata.AvailableSegmentMetadata; +import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; import org.apache.druid.segment.metadata.DataSourceInformation; -import org.apache.druid.segment.metadata.SegmentMetadataCache; import org.apache.druid.server.coordinator.CreateDataSegments; import org.apache.druid.server.coordinator.DruidCoordinator; import org.apache.druid.server.security.AuthConfig; @@ -140,7 +140,7 @@ public void testGetAllSegmentsWithOvershadowedStatus() @Test public void testGetAllSegmentsIncludingRealtime() { - SegmentMetadataCache segmentMetadataCache = Mockito.mock(SegmentMetadataCache.class); + CoordinatorSegmentMetadataCache coordinatorSegmentMetadataCache = Mockito.mock(CoordinatorSegmentMetadataCache.class); String dataSource2 = "datasource2"; @@ -205,14 +205,14 @@ public void testGetAllSegmentsIncludingRealtime() ).build() ); - Mockito.doReturn(availableSegments).when(segmentMetadataCache).getSegmentMetadataSnapshot(); + Mockito.doReturn(availableSegments).when(coordinatorSegmentMetadataCache).getSegmentMetadataSnapshot(); Mockito.doReturn(availableSegments.get(segments[0].getId())) - .when(segmentMetadataCache) + .when(coordinatorSegmentMetadataCache) .getAvailableSegmentMetadata(DATASOURCE1, segments[0].getId()); Mockito.doReturn(availableSegments.get(segments[1].getId())) - .when(segmentMetadataCache) + .when(coordinatorSegmentMetadataCache) .getAvailableSegmentMetadata(DATASOURCE1, segments[1].getId()); metadataResource = new MetadataResource( @@ -220,7 +220,7 @@ public void testGetAllSegmentsIncludingRealtime() storageCoordinator, AuthTestUtils.TEST_AUTHORIZER_MAPPER, coordinator, - segmentMetadataCache + coordinatorSegmentMetadataCache ); Response response = metadataResource.getAllUsedSegments(request, null, "includeOvershadowedStatus", "includeRealtimeSegments"); @@ -239,7 +239,7 @@ public void testGetAllSegmentsIncludingRealtime() @Test public void testGetDataSourceInformation() { - SegmentMetadataCache segmentMetadataCache = Mockito.mock(SegmentMetadataCache.class); + CoordinatorSegmentMetadataCache coordinatorSegmentMetadataCache = Mockito.mock(CoordinatorSegmentMetadataCache.class); Map dataSourceInformationMap = new HashMap<>(); dataSourceInformationMap.put( @@ -264,14 +264,14 @@ public void testGetDataSourceInformation() ) ); - Mockito.doReturn(dataSourceInformationMap).when(segmentMetadataCache).getDataSourceInformationMap(); + Mockito.doReturn(dataSourceInformationMap).when(coordinatorSegmentMetadataCache).getDataSourceInformationMap(); metadataResource = new MetadataResource( segmentsMetadataManager, storageCoordinator, AuthTestUtils.TEST_AUTHORIZER_MAPPER, coordinator, - segmentMetadataCache + coordinatorSegmentMetadataCache ); Response response = metadataResource.getDataSourceInformation(Collections.singletonList(DATASOURCE1)); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 200c3e71ce7c..c042e9de2409 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -88,7 +88,7 @@ import org.apache.druid.query.lookup.LookupSerdeModule; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; -import org.apache.druid.segment.metadata.SegmentMetadataCache; +import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.ClientQuerySegmentWalker; import org.apache.druid.server.audit.AuditManagerProvider; @@ -481,7 +481,7 @@ public void configure(Binder binder) binder.bind(CoordinatorTimeline.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); binder.bind(TimelineServerView.class).to(QueryableCoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, QueryableCoordinatorServerView.class); - LifecycleModule.register(binder, SegmentMetadataCache.class); + LifecycleModule.register(binder, CoordinatorSegmentMetadataCache.class); } } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index e761ecd702d6..9c5e9b5f5e24 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -31,7 +31,6 @@ import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; -import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.security.Escalator; import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java index 4ef9d63788e9..37a106bdd9ba 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java @@ -24,7 +24,6 @@ import org.apache.druid.query.TableDataSource; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; -import org.apache.druid.segment.metadata.DataSourceInformation; import org.apache.druid.server.SegmentManager; import org.apache.druid.sql.calcite.table.DatasourceTable.PhysicalDatasourceMetadata; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index a913dfcc9a5b..40a254f9439b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -93,10 +93,10 @@ public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon { - private final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); + private static final BrokerSegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = BrokerSegmentMetadataCacheConfig.create("PT1S"); // Timeout to allow (rapid) debugging, while not blocking tests with errors. private static final int WAIT_TIMEOUT_SECS = 6; - + private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); private BrokerSegmentMetadataCache runningSchema; private CountDownLatch buildTableLatch = new CountDownLatch(1); private CountDownLatch markDataSourceLatch = new CountDownLatch(1); @@ -106,8 +106,6 @@ public class BrokerSegmentMetadataCacheTest extends SegmentMetadataCacheCommon Set joinableDataSourceNames; JoinableFactory globalTableJoinable; - private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper(); - @Before public void setUp() throws Exception { From 25cdce6bec144332d11e8ecedfea86700abb5f99 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 00:32:03 +0530 Subject: [PATCH 35/36] Rename PhysicalDataSourceMetadataBuilder to PhysicalDataSourceMetadataFactory --- .../calcite/schema/BrokerSegmentMetadataCache.java | 10 +++++----- ...r.java => PhysicalDatasourceMetadataFactory.java} | 4 ++-- .../schema/BrokerSegmentMetadataCacheTest.java | 12 ++++++------ .../calcite/schema/DruidSchemaNoDataInitTest.java | 2 +- ...va => PhysicalDataSourceMetadataFactoryTest.java} | 10 +++++----- .../druid/sql/calcite/schema/SystemSchemaTest.java | 2 +- .../druid/sql/calcite/util/QueryFrameworkUtils.java | 4 ++-- 7 files changed, 22 insertions(+), 22 deletions(-) rename sql/src/main/java/org/apache/druid/sql/calcite/schema/{PhysicalDatasourceMetadataBuilder.java => PhysicalDatasourceMetadataFactory.java} (96%) rename sql/src/test/java/org/apache/druid/sql/calcite/schema/{PhysicalDataSourceMetadataBuilderTest.java => PhysicalDataSourceMetadataFactoryTest.java} (88%) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 9c5e9b5f5e24..7113e3fbce81 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -60,7 +60,7 @@ public class BrokerSegmentMetadataCache extends AbstractSegmentMetadataCache segmentsToRefresh, final Set da FutureUtils.getUnchecked(coordinatorClient.fetchDataSourceInformation(dataSourcesToQuery), true) .forEach(dataSourceInformation -> polledDataSourceMetadata.put( dataSourceInformation.getDataSource(), - physicalDatasourceMetadataBuilder.build( + dataSourceMetadataFactory.build( dataSourceInformation.getDataSource(), dataSourceInformation.getRowSignature() ) @@ -151,7 +151,7 @@ public void refresh(final Set segmentsToRefresh, final Set da return; } - final PhysicalDatasourceMetadata physicalDatasourceMetadata = physicalDatasourceMetadataBuilder.build(dataSource, rowSignature); + final PhysicalDatasourceMetadata physicalDatasourceMetadata = dataSourceMetadataFactory.build(dataSource, rowSignature); final PhysicalDatasourceMetadata oldTable = tables.put(dataSource, physicalDatasourceMetadata); if (oldTable == null || !oldTable.getRowSignature().equals(physicalDatasourceMetadata.getRowSignature())) { log.info("[%s] has new signature: %s.", dataSource, rowSignature); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataFactory.java similarity index 96% rename from sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java rename to sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataFactory.java index 37a106bdd9ba..106a0d401629 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataBuilder.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/PhysicalDatasourceMetadataFactory.java @@ -31,13 +31,13 @@ * Builds {@link PhysicalDatasourceMetadata} for a dataSource, including information about its schema, * joinability, and broadcast status. */ -public class PhysicalDatasourceMetadataBuilder +public class PhysicalDatasourceMetadataFactory { private final JoinableFactory joinableFactory; private final SegmentManager segmentManager; @Inject - public PhysicalDatasourceMetadataBuilder(JoinableFactory joinableFactory, SegmentManager segmentManager) + public PhysicalDatasourceMetadataFactory(JoinableFactory joinableFactory, SegmentManager segmentManager) { this.joinableFactory = joinableFactory; this.segmentManager = segmentManager; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java index 40a254f9439b..ad804b78e918 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCacheTest.java @@ -169,7 +169,7 @@ public BrokerSegmentMetadataCache buildSchemaMarkAndTableLatch(BrokerSegmentMeta new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), coordinatorClient ) { @@ -204,7 +204,7 @@ public BrokerSegmentMetadataCache buildSchemaMarkAndRefreshLatch() throws Interr new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), new NoopCoordinatorClient() ) { @@ -265,7 +265,7 @@ public ListenableFuture> fetchDataSourceInformation( new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), coordinatorClient ); @@ -328,7 +328,7 @@ public ListenableFuture> fetchDataSourceInformation( new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), coordinatorClient ); @@ -693,7 +693,7 @@ public void testRunSegmentMetadataQueryWithContext() throws Exception new NoopEscalator(), internalQueryConfig, new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), new NoopCoordinatorClient() ); @@ -725,7 +725,7 @@ public void testRefreshShouldEmitMetrics() throws InterruptedException, IOExcept new NoopEscalator(), new InternalQueryConfig(), emitter, - new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager), + new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager), new NoopCoordinatorClient() ) { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java index 2ac84ed76de1..754136d853d4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/DruidSchemaNoDataInitTest.java @@ -59,7 +59,7 @@ public void testInitializationWithNoData() throws Exception new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder( + new PhysicalDatasourceMetadataFactory( new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), new SegmentManager(EasyMock.createMock(SegmentLoader.class))), null diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataFactoryTest.java similarity index 88% rename from sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java rename to sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataFactoryTest.java index 3847bddd43ac..4700a387d0e3 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataBuilderTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/PhysicalDataSourceMetadataFactoryTest.java @@ -38,14 +38,14 @@ import java.util.Optional; import java.util.Set; -public class PhysicalDataSourceMetadataBuilderTest +public class PhysicalDataSourceMetadataFactoryTest { private Set segmentDataSourceNames; private Set joinableDataSourceNames; private SegmentManager segmentManager; private JoinableFactory globalTableJoinable; - private PhysicalDatasourceMetadataBuilder physicalDatasourceMetadataBuilder; + private PhysicalDatasourceMetadataFactory datasourceMetadataFactory; @Before public void setUp() @@ -80,7 +80,7 @@ public Optional build( } }; - physicalDatasourceMetadataBuilder = new PhysicalDatasourceMetadataBuilder(globalTableJoinable, segmentManager); + datasourceMetadataFactory = new PhysicalDatasourceMetadataFactory(globalTableJoinable, segmentManager); } @Test @@ -101,13 +101,13 @@ public void testBuild() .add("d2", ColumnType.DOUBLE) .build(); - DatasourceTable.PhysicalDatasourceMetadata fooDs = physicalDatasourceMetadataBuilder.build("foo", fooSignature); + DatasourceTable.PhysicalDatasourceMetadata fooDs = datasourceMetadataFactory.build("foo", fooSignature); Assert.assertTrue(fooDs.isJoinable()); Assert.assertTrue(fooDs.isBroadcast()); Assert.assertEquals(fooDs.dataSource().getName(), "foo"); Assert.assertEquals(fooDs.getRowSignature(), fooSignature); - DatasourceTable.PhysicalDatasourceMetadata barDs = physicalDatasourceMetadataBuilder.build("bar", barSignature); + DatasourceTable.PhysicalDatasourceMetadata barDs = datasourceMetadataFactory.build("bar", barSignature); Assert.assertFalse(barDs.isJoinable()); Assert.assertFalse(barDs.isBroadcast()); Assert.assertEquals(barDs.dataSource().getName(), "bar"); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index 4f0aaaa8e9df..46f6c5fda9b2 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -258,7 +258,7 @@ public void setUp() throws Exception new NoopEscalator(), new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder( + new PhysicalDatasourceMetadataFactory( new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), new SegmentManager(EasyMock.createMock(SegmentLoader.class)) ), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index 14e1912a93a9..0592d7de8afa 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -71,7 +71,7 @@ import org.apache.druid.sql.calcite.schema.NamedSystemSchema; import org.apache.druid.sql.calcite.schema.NamedViewSchema; import org.apache.druid.sql.calcite.schema.NoopDruidSchemaManager; -import org.apache.druid.sql.calcite.schema.PhysicalDatasourceMetadataBuilder; +import org.apache.druid.sql.calcite.schema.PhysicalDatasourceMetadataFactory; import org.apache.druid.sql.calcite.schema.SystemSchema; import org.apache.druid.sql.calcite.schema.ViewSchema; import org.apache.druid.sql.calcite.view.ViewManager; @@ -218,7 +218,7 @@ private static DruidSchema createMockSchema( CalciteTests.TEST_AUTHENTICATOR_ESCALATOR, new InternalQueryConfig(), new NoopServiceEmitter(), - new PhysicalDatasourceMetadataBuilder( + new PhysicalDatasourceMetadataFactory( createDefaultJoinableFactory(injector), new SegmentManager(EasyMock.createMock(SegmentLoader.class)) { From 5f5ad18202d4635d890693ee637c963325edf257 Mon Sep 17 00:00:00 2001 From: rishabh singh Date: Thu, 14 Sep 2023 01:58:29 +0530 Subject: [PATCH 36/36] Fix json property key name in DataSourceInformation --- .../client/QueryableCoordinatorServerView.java | 5 ++--- .../CoordinatorSegmentMetadataCache.java | 2 +- .../segment/metadata/DataSourceInformation.java | 2 +- .../org/apache/druid/cli/CliCoordinator.java | 17 +++++++++-------- .../schema/BrokerSegmentMetadataCache.java | 7 ++++--- 5 files changed, 17 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java index 1db03afa2f85..96d5d80eee72 100644 --- a/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java +++ b/server/src/main/java/org/apache/druid/client/QueryableCoordinatorServerView.java @@ -32,7 +32,6 @@ import org.apache.druid.query.DataSource; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.QueryWatcher; -import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.VersionedIntervalTimeline; @@ -50,7 +49,7 @@ * of {@link ServerSelector} objects, while the other class stores {@link SegmentLoadInfo} object in its timeline.

    * *

    A new timeline class (implementing {@link TimelineServerView}) is required for - * {@link AbstractSegmentMetadataCache}, which will run on the Coordinator.

    + * {@link org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache}, which will run on the Coordinator.

    */ @ManageLifecycle public class QueryableCoordinatorServerView extends BrokerServerView implements CoordinatorTimeline @@ -81,7 +80,7 @@ public boolean isAwaitInitializationOnStart() /** * This class maintains a timeline of {@link ServerSelector} objects. - * This method returns a new timeline of the object {@link SegmentLoadInfo}. + * This method converts and returns a new timeline of the object {@link SegmentLoadInfo}. * * @param dataSource dataSoruce * @return timeline for the given dataSource diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java index a24308ff4723..15bb1f7c1fab 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java @@ -41,7 +41,7 @@ @ManageLifecycle public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCache { - private static final EmittingLogger log = new EmittingLogger(AbstractSegmentMetadataCache.class); + private static final EmittingLogger log = new EmittingLogger(CoordinatorSegmentMetadataCache.class); @Inject public CoordinatorSegmentMetadataCache( diff --git a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java index f25c104203c5..e007fdb9af18 100644 --- a/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java +++ b/server/src/main/java/org/apache/druid/segment/metadata/DataSourceInformation.java @@ -36,7 +36,7 @@ public class DataSourceInformation @JsonCreator public DataSourceInformation( - @JsonProperty("datasource") String dataSource, + @JsonProperty("dataSource") String dataSource, @JsonProperty("rowSignature") RowSignature rowSignature) { this.dataSource = Preconditions.checkNotNull(dataSource, "'dataSource' must be nonnull"); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index c042e9de2409..d60bc67e4ce1 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -87,7 +87,6 @@ import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.lookup.LookupSerdeModule; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; -import org.apache.druid.segment.metadata.AbstractSegmentMetadataCache; import org.apache.druid.segment.metadata.CoordinatorSegmentMetadataCache; import org.apache.druid.segment.metadata.SegmentMetadataCacheConfig; import org.apache.druid.server.ClientQuerySegmentWalker; @@ -221,11 +220,11 @@ public void configure(Binder binder) } if (isSegmentMetadataCacheEnabled()) { - binder.install(new SegmentMetadataCacheModule()); + binder.install(new CoordinatorSegmentMetadataCacheModule()); } else { binder.bind(CoordinatorTimeline.class).to(CoordinatorServerView.class).in(LazySingleton.class); LifecycleModule.register(binder, CoordinatorServerView.class); - binder.bind(AbstractSegmentMetadataCache.class).toProvider(Providers.of(null)); + binder.bind(CoordinatorSegmentMetadataCache.class).toProvider(Providers.of(null)); } binder.bind(SegmentsMetadataManager.class) @@ -457,11 +456,13 @@ public Supplier> get() } } - private static class SegmentMetadataCacheModule implements Module + private static class CoordinatorSegmentMetadataCacheModule implements Module { @Override public void configure(Binder binder) { + // These modules are required to allow running queries on the Coordinator, + // since CoordinatorSegmentMetadataCache needs to query data nodes and tasks binder.install(new LegacyBrokerParallelMergeConfigModule()); binder.install(new QueryRunnerFactoryModule()); binder.install(new SegmentWranglerModule()); @@ -470,10 +471,10 @@ public void configure(Binder binder) binder.install(new JoinableFactoryModule()); JsonConfigProvider.bind(binder, "druid.coordinator.internal.query.config", InternalQueryConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.select", TierSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.broker.select.tier.custom", CustomTierSelectorStrategyConfig.class); - JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); - JsonConfigProvider.bind(binder, "druid.broker.retryPolicy", RetryQueryRunnerConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.select", TierSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.coordinator.select.tier.custom", CustomTierSelectorStrategyConfig.class); + JsonConfigProvider.bind(binder, "druid.coordinator.balancer", ServerSelectorStrategy.class); + JsonConfigProvider.bind(binder, "druid.coordinator.retryPolicy", RetryQueryRunnerConfig.class); binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); binder.bind(CachingClusteredClient.class).in(LazySingleton.class); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java index 7113e3fbce81..42716f436b81 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java @@ -19,9 +19,9 @@ package org.apache.druid.sql.calcite.schema; +import com.google.common.base.Predicates; import com.google.common.collect.Sets; import com.google.inject.Inject; -import io.vavr.Predicates; import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.client.TimelineServerView; import org.apache.druid.client.coordinator.CoordinatorClient; @@ -90,8 +90,9 @@ public BrokerSegmentMetadataCache( /** * Refreshes the set of segments in two steps: *
      - *
    • Polls the coordinator for the dataSource schema to update the {@code tables}.
      • - *
    • Refreshes the remaining set of segments by executing a SegmentMetadataQuery.
      • + *
    • Polls the coordinator for the dataSource schema.
      • + *
    • Refreshes the remaining set of segments by executing a SegmentMetadataQuery and + * builds dataSource schema by combining segment schema.
      • *
    */ @Override