From f3d6422ae289fc3821e07c4ef9ed5810d67c9f64 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 20 Oct 2020 14:07:21 +0800 Subject: [PATCH 01/47] druid task auto scale based on kafka lag --- .../kafka/supervisor/KafkaSupervisor.java | 7 + .../kinesis/supervisor/KinesisSupervisor.java | 5 + .../supervisor/SupervisorManager.java | 6 + .../supervisor/SeekableStreamSupervisor.java | 326 ++++++++++++++++-- .../SeekableStreamSupervisorIOConfig.java | 19 +- .../SeekableStreamSupervisorStateTest.java | 13 +- 6 files changed, 353 insertions(+), 23 deletions(-) diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java index 021d91664f1b..8f87e023c876 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java @@ -57,6 +57,7 @@ import org.joda.time.DateTime; import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -329,6 +330,12 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() return false; } + @Override + protected void collectLag(ArrayList lags) + { + computeLags(getPartitionRecordLag(), lags); + } + @Override protected void updatePartitionLagFromStream() { diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java index ba5129a3d93d..a1f17c5da512 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java @@ -377,6 +377,11 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() return true; } + @Override + protected void collectLag(ArrayList lags) + { + } + @Override protected Map> filterExpiredPartitionsFromStartingOffsets( Map> startingOffsets diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java index 48153b086ff3..5e808e38936f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java @@ -30,6 +30,7 @@ import org.apache.druid.metadata.MetadataSupervisorManager; import javax.annotation.Nullable; + import java.util.List; import java.util.Map; import java.util.Set; @@ -54,6 +55,11 @@ public SupervisorManager(MetadataSupervisorManager metadataSupervisorManager) this.metadataSupervisorManager = metadataSupervisorManager; } + public MetadataSupervisorManager getMetadataSupervisorManager() + { + return metadataSupervisorManager; + } + public Set getSupervisorIds() { return supervisors.keySet(); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index da6dc8b83ad0..4b3e8a364040 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -40,6 +40,7 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.collections4.queue.CircularFifoQueue; import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.TaskStatus; @@ -55,6 +56,7 @@ import org.apache.druid.indexing.overlord.TaskRunnerWorkItem; import org.apache.druid.indexing.overlord.TaskStorage; import org.apache.druid.indexing.overlord.supervisor.Supervisor; +import org.apache.druid.indexing.overlord.supervisor.SupervisorManager; import org.apache.druid.indexing.overlord.supervisor.SupervisorReport; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager; import org.apache.druid.indexing.seekablestream.SeekableStreamDataSourceMetadata; @@ -81,12 +83,14 @@ import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.metadata.EntryExistsException; +import org.apache.druid.metadata.MetadataSupervisorManager; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; import org.apache.druid.segment.indexing.DataSchema; import org.joda.time.DateTime; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; + import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -143,6 +147,7 @@ public abstract class SeekableStreamSupervisor lags = collectTotalLags(); + boolean allocationSuccess = dynamicAllocate(lags); + if (allocationSuccess) { + dynamicTriggerLastRunTime = nowTime; + queue.clear(); + } + } + catch (Exception e) { + log.error(e, "Error, when parse DynamicAllocationTasksNotice"); + } + finally { + lock.unlock(); + } + } + } + + private boolean dynamicAllocate(List lags) throws InterruptedException, ExecutionException, TimeoutException + { + // if supervisor is not suspended, ensure required tasks are running + // if suspended, ensure tasks have been requested to gracefully stop + log.info("[%s] supervisor is running, start to check dynamic allocate task logic", dataSource); + long scaleOutThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutThreshold", 5000000))); + long scaleInThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInThreshold", 1000000))); + double triggerSaleOutThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerSaleOutThresholdFrequency", 0.3))); + double triggerSaleInThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerSaleInThresholdFrequency", 0.8))); + int taskCountMax = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 8))); + int taskCountMin = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMin", 1))); + int scaleInStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInStep", 1))); + int scaleOutStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutStep", 2))); + int beyond = 0; + int within = 0; + int metricsCount = lags.size(); + for (Long lag : lags) { + if (lag >= scaleOutThreshold) { + beyond++; + } + if (lag <= scaleInThreshold) { + within++; + } + } + double beyondProportion = beyond * 1.0 / metricsCount; + double withinProportion = within * 1.0 / metricsCount; + log.info("triggerSaleOutThresholdFrequency is [ " + triggerSaleOutThresholdFrequency + " ] and triggerSaleInThresholdFrequency is [ " + triggerSaleInThresholdFrequency + " ]"); + log.info("beyondProportion is [ " + beyondProportion + " ] and withinProportion is [ " + withinProportion + " ]"); + + int currentActiveTaskCount = 0; + int desireActiveTaskCount; + Collection activeTaskGroups = activelyReadingTaskGroups.values(); + currentActiveTaskCount = activeTaskGroups.size(); + + if (beyondProportion >= triggerSaleOutThresholdFrequency) { + // Do Scale out + int taskCount = currentActiveTaskCount + scaleOutStep; + if (currentActiveTaskCount == taskCountMax) { + log.info("CurrentActiveTaskCount reach task count Max limit, skip to scale out tasks"); + return false; + } else { + desireActiveTaskCount = Math.min(taskCount, taskCountMax); + } + log.info("Start to scale out tasks , current active task number [ " + currentActiveTaskCount + " ] and desire task number is [ " + desireActiveTaskCount + " ] "); + gracefulShutdownInternal(); + // clear everything + clearAllocationInfos(); + log.info("Set Task Count : " + desireActiveTaskCount); + setTaskCount(desireActiveTaskCount); + return true; + } + + if (withinProportion >= triggerSaleInThresholdFrequency) { + // Do Scale in + int taskCount = currentActiveTaskCount - scaleInStep; + if (currentActiveTaskCount == taskCountMin) { + log.info("CurrentActiveTaskCount reach task count Min limit, skip to scale in tasks"); + return false; + } else { + desireActiveTaskCount = Math.max(taskCount, taskCountMin); + } + log.info("Start to scale in tasks , current active task number [ " + currentActiveTaskCount + " ] and desire task number is [ " + desireActiveTaskCount + " ] "); + gracefulShutdownInternal(); + // clear everything + clearAllocationInfos(); + log.info("Set Task Count : " + desireActiveTaskCount); + setTaskCount(desireActiveTaskCount); + return true; + } + return false; + } + + private void setTaskCount(int desireActiveTaskCount) + { + ioConfig.setTaskCount(desireActiveTaskCount); + try { + Optional supervisorManager = taskMaster.getSupervisorManager(); + if (supervisorManager.isPresent()) { + MetadataSupervisorManager metadataSupervisorManager = supervisorManager.get().getMetadataSupervisorManager(); + metadataSupervisorManager.insert(dataSource, spec); + } else { + log.error("supervisorManager is null in taskMaster, skip to do scale action"); + } + } + catch (Exception e) { + log.error("Failed to sync taskCount to RDS"); + } + } + + private void clearAllocationInfos() + { + activelyReadingTaskGroups.clear(); + partitionGroups.clear(); + partitionOffsets.clear(); + + pendingCompletionTaskGroups.clear(); + partitionIds.clear(); + } + + private List collectTotalLags() + { + return queue.stream().collect(Collectors.toList()); + } + private class GracefulShutdownNotice extends ShutdownNotice { @Override @@ -469,6 +625,7 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private final SeekableStreamIndexTaskClient taskClient; private final SeekableStreamSupervisorSpec spec; private final SeekableStreamSupervisorIOConfig ioConfig; + private final Map dynamicAllocationTasksProperties; private final SeekableStreamSupervisorTuningConfig tuningConfig; private final SeekableStreamIndexTaskTuningConfig taskTuningConfig; private final String supervisorId; @@ -478,6 +635,8 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private final ExecutorService exec; private final ScheduledExecutorService scheduledExec; private final ScheduledExecutorService reportingExec; + private final ScheduledExecutorService allocationExec; + private final ScheduledExecutorService lagComputationExec; private final ListeningExecutorService workerExec; private final BlockingQueue notices = new LinkedBlockingDeque<>(); private final Object stopLock = new Object(); @@ -487,6 +646,7 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private final boolean useExclusiveStartingSequence; private boolean listenerRegistered = false; private long lastRunTime; + private long dynamicTriggerLastRunTime; private int initRetryCounter = 0; private volatile DateTime firstRunTime; private volatile DateTime earlyStopTime = null; @@ -495,6 +655,12 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private volatile boolean stopped = false; private volatile boolean lifecycleStarted = false; private final ServiceEmitter emitter; + private final boolean enableDynamicAllocationTasks; + private volatile long metricsCollectionIntervalMillis; + private volatile long metricsCollectionRangeMillis; + private volatile long dynamicCheckStartDelayMillis; + private volatile long dynamicCheckPeriod; + private volatile CircularFifoQueue queue; public SeekableStreamSupervisor( final String supervisorId, @@ -518,20 +684,52 @@ public SeekableStreamSupervisor( this.useExclusiveStartingSequence = useExclusiveStartingSequence; this.dataSource = spec.getDataSchema().getDataSource(); this.ioConfig = spec.getIoConfig(); + this.dynamicAllocationTasksProperties = ioConfig.getDynamicAllocationTasksProperties(); + log.info("Get dynamicAllocationTasksProperties from IOConfig : " + dynamicAllocationTasksProperties); + + if (dynamicAllocationTasksProperties != null && !dynamicAllocationTasksProperties.isEmpty() && Boolean.parseBoolean(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("enableDynamicAllocationTasks", false)))) { + log.info("EnableDynamicAllocationTasks for datasource " + dataSource); + this.enableDynamicAllocationTasks = true; + } else { + log.info("Disable Dynamic Allocate Tasks"); + this.enableDynamicAllocationTasks = false; + } + int taskCountMax = 0; + if (enableDynamicAllocationTasks) { + this.metricsCollectionIntervalMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionIntervalMillis", 10000))); + this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionRangeMillis", 6 * 10 * 1000))); + int slots = (int) (metricsCollectionRangeMillis / metricsCollectionIntervalMillis) + 1; + log.info(" The interval of metrics collection is " + metricsCollectionIntervalMillis + ", " + metricsCollectionRangeMillis + " timeRange will collect " + slots + " data points at most."); + this.queue = new CircularFifoQueue<>(slots); + taskCountMax = Integer.parseInt(String.valueOf(this.dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 8))); + this.dynamicCheckStartDelayMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckStartDelayMillis", 300000))); + this.dynamicCheckPeriod = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckPeriod", 600000))); + this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionRangeMillis", 600000))); + } + this.tuningConfig = spec.getTuningConfig(); this.taskTuningConfig = this.tuningConfig.convertToTaskTuningConfig(); this.supervisorId = supervisorId; this.exec = Execs.singleThreaded(supervisorId); this.scheduledExec = Execs.scheduledSingleThreaded(supervisorId + "-Scheduler-%d"); this.reportingExec = Execs.scheduledSingleThreaded(supervisorId + "-Reporting-%d"); + this.allocationExec = Execs.scheduledSingleThreaded(supervisorId + "-Allocation-%d"); + this.lagComputationExec = Execs.scheduledSingleThreaded(supervisorId + "-Computation-%d"); this.stateManager = new SeekableStreamSupervisorStateManager( spec.getSupervisorStateManagerConfig(), spec.isSuspended() ); - int workerThreads = (this.tuningConfig.getWorkerThreads() != null - ? this.tuningConfig.getWorkerThreads() - : Math.min(10, this.ioConfig.getTaskCount())); + int workerThreads = 0; + if (enableDynamicAllocationTasks) { + workerThreads = (this.tuningConfig.getWorkerThreads() != null + ? this.tuningConfig.getWorkerThreads() + : Math.min(10, taskCountMax)); + } else { + workerThreads = (this.tuningConfig.getWorkerThreads() != null + ? this.tuningConfig.getWorkerThreads() + : Math.min(10, this.ioConfig.getTaskCount())); + } this.workerExec = MoreExecutors.listeningDecorator(Execs.multiThreaded(workerThreads, supervisorId + "-Worker-%d")); log.info("Created worker pool with [%d] threads for dataSource [%s]", workerThreads, this.dataSource); @@ -571,10 +769,17 @@ public Optional getTaskStatus(String id) tuningConfig.getChatRetries() * (tuningConfig.getHttpTimeout().getStandardSeconds() + IndexTaskClient.MAX_RETRY_WAIT_SECONDS) ); + int chatThreads; + if (enableDynamicAllocationTasks) { + chatThreads = (this.tuningConfig.getChatThreads() != null + ? this.tuningConfig.getChatThreads() + : Math.min(10, taskCountMax * this.ioConfig.getReplicas())); + } else { + chatThreads = (this.tuningConfig.getChatThreads() != null + ? this.tuningConfig.getChatThreads() + : Math.min(10, this.ioConfig.getTaskCount() * this.ioConfig.getReplicas())); + } - int chatThreads = (this.tuningConfig.getChatThreads() != null - ? this.tuningConfig.getChatThreads() - : Math.min(10, this.ioConfig.getTaskCount() * this.ioConfig.getReplicas())); this.taskClient = taskClientFactory.build( taskInfoProvider, dataSource, @@ -652,6 +857,11 @@ public void stop(boolean stopGracefully) try { scheduledExec.shutdownNow(); // stop recurring executions reportingExec.shutdownNow(); + log.info("Shut Down allocationExec now"); + allocationExec.shutdownNow(); + log.info("Shut Down lagComputationExec now"); + lagComputationExec.shutdownNow(); + if (started) { Optional taskRunner = taskMaster.getTaskRunner(); @@ -768,7 +978,22 @@ public void tryInit() ); scheduleReporting(reportingExec); - + if (enableDynamicAllocationTasks) { + log.info("Collect and compute lags at fixed rate of " + metricsCollectionIntervalMillis); + lagComputationExec.scheduleAtFixedRate( + collectAndcollectLags(), + dynamicCheckStartDelayMillis, // wait for tasks to start up + metricsCollectionIntervalMillis, + TimeUnit.MILLISECONDS + ); + log.info("allocate task at fixed rate of " + dynamicCheckPeriod); + allocationExec.scheduleAtFixedRate( + buildDynamicAllocationTask(), + dynamicCheckStartDelayMillis + metricsCollectionRangeMillis, + dynamicCheckPeriod, + TimeUnit.MILLISECONDS + ); + } started = true; log.info( "Started SeekableStreamSupervisor[%s], first run in [%s], with spec: [%s]", @@ -791,6 +1016,38 @@ public void tryInit() } } + private Runnable collectAndcollectLags() + { + return new Runnable() { + @Override + public void run() + { + lock.lock(); + try { + if (!spec.isSuspended()) { + ArrayList metricsInfo = new ArrayList<>(3); + collectLag(metricsInfo); + long totalLags = metricsInfo.size() < 3 ? 0 : metricsInfo.get(1); + queue.offer(totalLags > 0 ? totalLags : 0); + log.info("Current lag metric points : " + queue.stream().collect(Collectors.toList())); + } else { + log.info("[%s] supervisor is suspended, skip to collect kafka lags", dataSource); + } + } + catch (Exception e) { + log.error(e, "Error, When collect kafka lags"); + } + finally { + lock.unlock(); + } + } + }; + } + private Runnable buildDynamicAllocationTask() + { + return () -> notices.add(new DynamicAllocationTasksNotice()); + } + private Runnable buildRunTask() { return () -> notices.add(new RunNotice()); @@ -1137,6 +1394,20 @@ public void gracefulShutdownInternal() throws ExecutionException, InterruptedExc @VisibleForTesting public void resetInternal(DataSourceMetadata dataSourceMetadata) { + // clear queue for kafka lags + if (enableDynamicAllocationTasks && queue != null) { + try { + lock.lock(); + queue.clear(); + } + catch (Exception e) { + log.warn(e, "Error,when clear queue in rest action"); + } + finally { + lock.unlock(); + } + } + if (dataSourceMetadata == null) { // Reset everything boolean result = indexerMetadataStorageCoordinator.deleteDataSourceMetadata(dataSource); @@ -3487,33 +3758,25 @@ protected void emitLag() final String type = spec.getType(); BiConsumer, String> emitFn = (partitionLags, suffix) -> { - if (partitionLags == null) { + ArrayList lags = new ArrayList<>(3); + computeLags(partitionLags, lags); + if (lags.size() < 3) { return; } - - long maxLag = 0, totalLag = 0, avgLag; - for (long lag : partitionLags.values()) { - if (lag > maxLag) { - maxLag = lag; - } - totalLag += lag; - } - avgLag = partitionLags.size() == 0 ? 0 : totalLag / partitionLags.size(); - emitter.emit( ServiceMetricEvent.builder() .setDimension("dataSource", dataSource) - .build(StringUtils.format("ingest/%s/lag%s", type, suffix), totalLag) + .build(StringUtils.format("ingest/%s/lag%s", type, suffix), lags.get(1)) ); emitter.emit( ServiceMetricEvent.builder() .setDimension("dataSource", dataSource) - .build(StringUtils.format("ingest/%s/maxLag%s", type, suffix), maxLag) + .build(StringUtils.format("ingest/%s/maxLag%s", type, suffix), lags.get(0)) ); emitter.emit( ServiceMetricEvent.builder() .setDimension("dataSource", dataSource) - .build(StringUtils.format("ingest/%s/avgLag%s", type, suffix), avgLag) + .build(StringUtils.format("ingest/%s/avgLag%s", type, suffix), lags.get(2)) ); }; @@ -3526,6 +3789,25 @@ protected void emitLag() } } + + protected void computeLags(Map partitionLags, ArrayList lags) + { + if (partitionLags == null) { + return; + } + long maxLag = 0, totalLag = 0, avgLag; + for (long lag : partitionLags.values()) { + if (lag > maxLag) { + maxLag = lag; + } + totalLag += lag; + } + avgLag = partitionLags.size() == 0 ? 0 : totalLag / partitionLags.size(); + lags.add(maxLag); + lags.add(totalLag); + lags.add(avgLag); + } + /** * a special sequence number that is used to indicate that the sequence offset * for a particular partition has not yet been calculated by the supervisor. When @@ -3561,4 +3843,6 @@ protected void emitLag() * sequences. In Kafka, start offsets are always inclusive. */ protected abstract boolean useExclusiveStartSequenceNumberForNonFirstSequence(); + + protected abstract void collectLag(ArrayList lags); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java index 723e22ec5183..31dc360f1705 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java @@ -30,6 +30,8 @@ import javax.annotation.Nullable; +import java.util.Map; + public abstract class SeekableStreamSupervisorIOConfig { @@ -37,7 +39,7 @@ public abstract class SeekableStreamSupervisorIOConfig @Nullable private final InputFormat inputFormat; // nullable for backward compatibility private final Integer replicas; - private final Integer taskCount; + private Integer taskCount; private final Duration taskDuration; private final Duration startDelay; private final Duration period; @@ -46,6 +48,7 @@ public abstract class SeekableStreamSupervisorIOConfig private final Optional lateMessageRejectionPeriod; private final Optional earlyMessageRejectionPeriod; private final Optional lateMessageRejectionStartDateTime; + private final Map dynamicAllocationTasksProperties; public SeekableStreamSupervisorIOConfig( String stream, @@ -59,6 +62,7 @@ public SeekableStreamSupervisorIOConfig( Period completionTimeout, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, + Map dynamicAllocationTasksProperties, DateTime lateMessageRejectionStartDateTime ) { @@ -87,6 +91,8 @@ public SeekableStreamSupervisorIOConfig( + "both properties lateMessageRejectionStartDateTime " + "and lateMessageRejectionPeriod."); } + // Could be null + this.dynamicAllocationTasksProperties = dynamicAllocationTasksProperties; } private static Duration defaultDuration(final Period period, final String theDefault) @@ -113,12 +119,23 @@ public Integer getReplicas() return replicas; } + @JsonProperty + public Map getDynamicAllocationTasksProperties() + { + return dynamicAllocationTasksProperties; + } + @JsonProperty public Integer getTaskCount() { return taskCount; } + public void setTaskCount(final int taskCount) + { + this.taskCount = taskCount; + } + @JsonProperty public Duration getTaskDuration() { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index 6bfbd7f9983f..c21eeeee22e7 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -764,6 +764,7 @@ private void expectEmitterSupervisor(boolean suspended) throws EntryExistsExcept new Period("PT30M"), null, null, + null, null ) { @@ -824,7 +825,7 @@ private static SeekableStreamSupervisorIOConfig getIOConfig() false, new Period("PT30M"), null, - null, null + null, null, null ) { }; @@ -1174,6 +1175,11 @@ protected void scheduleReporting(ScheduledExecutorService reportingExec) { // do nothing } + + @Override + protected void collectLag(ArrayList lags) + { + } } private class TestEmittingTestSeekableStreamSupervisor extends BaseTestSeekableStreamSupervisor @@ -1216,6 +1222,11 @@ protected void emitLag() } } + @Override + protected void collectLag(ArrayList lags) + { + } + @Override protected void scheduleReporting(ScheduledExecutorService reportingExec) { From 5c1c21c44c2737fe0e55405f863cfd8c93329213 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 21 Oct 2020 11:25:00 +0800 Subject: [PATCH 02/47] fix kafkaSupervisorIOConfig and KinesisSupervisorIOConfig --- .idea/misc.xml | 4 ++-- .../indexing/kafka/supervisor/KafkaSupervisorIOConfig.java | 2 ++ .../apache/druid/indexing/kafka/KafkaSamplerSpecTest.java | 1 + .../druid/indexing/kafka/supervisor/KafkaSupervisorTest.java | 3 +++ .../kinesis/supervisor/KinesisSupervisorIOConfig.java | 5 +++++ .../druid/indexing/kinesis/KinesisSamplerSpecTest.java | 1 + .../indexing/kinesis/supervisor/KinesisSupervisorTest.java | 5 +++++ 7 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java index a1360b5c63f3..a2653d2ce025 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java @@ -50,6 +50,7 @@ public KafkaSupervisorIOConfig( @JsonProperty("taskCount") Integer taskCount, @JsonProperty("taskDuration") Period taskDuration, @JsonProperty("consumerProperties") Map consumerProperties, + @JsonProperty("dynamicAllocationTasksProperties") Map dynamicAllocationTasksProperties, @JsonProperty("pollTimeout") Long pollTimeout, @JsonProperty("startDelay") Period startDelay, @JsonProperty("period") Period period, @@ -72,6 +73,7 @@ public KafkaSupervisorIOConfig( completionTimeout, lateMessageRejectionPeriod, earlyMessageRejectionPeriod, + dynamicAllocationTasksProperties, lateMessageRejectionStartDateTime ); diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java index 3768cebd62da..22553dec571d 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java @@ -134,6 +134,7 @@ public void testSample() null, null, null, + null, true, null, null, diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 905abd19e2ac..be402652b8a2 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -3372,6 +3372,7 @@ private TestableKafkaSupervisor getTestableSupervisor( taskCount, new Period(duration), consumerProperties, + null, KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), @@ -3482,6 +3483,7 @@ private TestableKafkaSupervisor getTestableSupervisorCustomIsTaskCurrent( taskCount, new Period(duration), consumerProperties, + null, KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), @@ -3596,6 +3598,7 @@ private KafkaSupervisor getSupervisor( taskCount, new Period(duration), consumerProperties, + null, KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java index f68e0b7a09b3..885fb6e9f7dd 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java @@ -29,6 +29,8 @@ import org.joda.time.DateTime; import org.joda.time.Period; +import java.util.Map; + public class KinesisSupervisorIOConfig extends SeekableStreamSupervisorIOConfig { private final String endpoint; @@ -70,6 +72,7 @@ public KinesisSupervisorIOConfig( @JsonProperty("fetchDelayMillis") Integer fetchDelayMillis, @JsonProperty("awsAssumedRoleArn") String awsAssumedRoleArn, @JsonProperty("awsExternalId") String awsExternalId, + @JsonProperty("dynamicAllocationTasksProperties") Map dynamicAllocationTasksProperties, @JsonProperty("deaggregate") boolean deaggregate ) { @@ -85,7 +88,9 @@ public KinesisSupervisorIOConfig( completionTimeout, lateMessageRejectionPeriod, earlyMessageRejectionPeriod, + dynamicAllocationTasksProperties, lateMessageRejectionStartDateTime + ); this.endpoint = endpoint != null ? endpoint diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java index dadc6c7a944d..0042b0278f84 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java @@ -152,6 +152,7 @@ public void testSample() throws Exception null, null, null, + null, false ), null, diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java index 135166250d99..47871e71fa80 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java @@ -301,6 +301,7 @@ public void testRecordSupplier() 1000, null, null, + null, false ); KinesisIndexTaskClientFactory clientFactory = new KinesisIndexTaskClientFactory(null, OBJECT_MAPPER); @@ -4716,6 +4717,7 @@ private TestableKinesisSupervisor getTestableSupervisor( null, null, null, + null, false ); @@ -4856,6 +4858,7 @@ private TestableKinesisSupervisor getTestableSupervisor( fetchDelayMillis, null, null, + null, false ); @@ -4943,6 +4946,7 @@ private TestableKinesisSupervisor getTestableSupervisorCustomIsTaskCurrent( fetchDelayMillis, null, null, + null, false ); @@ -5032,6 +5036,7 @@ private KinesisSupervisor getSupervisor( fetchDelayMillis, null, null, + null, false ); From 6d7582be1e9c423a50085b57a3a163a5166316f4 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 20 Oct 2020 14:07:21 +0800 Subject: [PATCH 03/47] druid task auto scale based on kafka lag --- .../kafka/supervisor/KafkaSupervisor.java | 7 + .../kinesis/supervisor/KinesisSupervisor.java | 5 + .../supervisor/SupervisorManager.java | 6 + .../supervisor/SeekableStreamSupervisor.java | 326 ++++++++++++++++-- .../SeekableStreamSupervisorIOConfig.java | 19 +- .../SeekableStreamSupervisorStateTest.java | 13 +- 6 files changed, 353 insertions(+), 23 deletions(-) diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java index 021d91664f1b..8f87e023c876 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java @@ -57,6 +57,7 @@ import org.joda.time.DateTime; import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -329,6 +330,12 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() return false; } + @Override + protected void collectLag(ArrayList lags) + { + computeLags(getPartitionRecordLag(), lags); + } + @Override protected void updatePartitionLagFromStream() { diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java index ba5129a3d93d..a1f17c5da512 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java @@ -377,6 +377,11 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() return true; } + @Override + protected void collectLag(ArrayList lags) + { + } + @Override protected Map> filterExpiredPartitionsFromStartingOffsets( Map> startingOffsets diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java index 48153b086ff3..5e808e38936f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java @@ -30,6 +30,7 @@ import org.apache.druid.metadata.MetadataSupervisorManager; import javax.annotation.Nullable; + import java.util.List; import java.util.Map; import java.util.Set; @@ -54,6 +55,11 @@ public SupervisorManager(MetadataSupervisorManager metadataSupervisorManager) this.metadataSupervisorManager = metadataSupervisorManager; } + public MetadataSupervisorManager getMetadataSupervisorManager() + { + return metadataSupervisorManager; + } + public Set getSupervisorIds() { return supervisors.keySet(); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index da6dc8b83ad0..4b3e8a364040 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -40,6 +40,7 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.collections4.queue.CircularFifoQueue; import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskState; import org.apache.druid.indexer.TaskStatus; @@ -55,6 +56,7 @@ import org.apache.druid.indexing.overlord.TaskRunnerWorkItem; import org.apache.druid.indexing.overlord.TaskStorage; import org.apache.druid.indexing.overlord.supervisor.Supervisor; +import org.apache.druid.indexing.overlord.supervisor.SupervisorManager; import org.apache.druid.indexing.overlord.supervisor.SupervisorReport; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager; import org.apache.druid.indexing.seekablestream.SeekableStreamDataSourceMetadata; @@ -81,12 +83,14 @@ import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.metadata.EntryExistsException; +import org.apache.druid.metadata.MetadataSupervisorManager; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; import org.apache.druid.segment.indexing.DataSchema; import org.joda.time.DateTime; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; + import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -143,6 +147,7 @@ public abstract class SeekableStreamSupervisor lags = collectTotalLags(); + boolean allocationSuccess = dynamicAllocate(lags); + if (allocationSuccess) { + dynamicTriggerLastRunTime = nowTime; + queue.clear(); + } + } + catch (Exception e) { + log.error(e, "Error, when parse DynamicAllocationTasksNotice"); + } + finally { + lock.unlock(); + } + } + } + + private boolean dynamicAllocate(List lags) throws InterruptedException, ExecutionException, TimeoutException + { + // if supervisor is not suspended, ensure required tasks are running + // if suspended, ensure tasks have been requested to gracefully stop + log.info("[%s] supervisor is running, start to check dynamic allocate task logic", dataSource); + long scaleOutThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutThreshold", 5000000))); + long scaleInThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInThreshold", 1000000))); + double triggerSaleOutThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerSaleOutThresholdFrequency", 0.3))); + double triggerSaleInThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerSaleInThresholdFrequency", 0.8))); + int taskCountMax = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 8))); + int taskCountMin = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMin", 1))); + int scaleInStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInStep", 1))); + int scaleOutStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutStep", 2))); + int beyond = 0; + int within = 0; + int metricsCount = lags.size(); + for (Long lag : lags) { + if (lag >= scaleOutThreshold) { + beyond++; + } + if (lag <= scaleInThreshold) { + within++; + } + } + double beyondProportion = beyond * 1.0 / metricsCount; + double withinProportion = within * 1.0 / metricsCount; + log.info("triggerSaleOutThresholdFrequency is [ " + triggerSaleOutThresholdFrequency + " ] and triggerSaleInThresholdFrequency is [ " + triggerSaleInThresholdFrequency + " ]"); + log.info("beyondProportion is [ " + beyondProportion + " ] and withinProportion is [ " + withinProportion + " ]"); + + int currentActiveTaskCount = 0; + int desireActiveTaskCount; + Collection activeTaskGroups = activelyReadingTaskGroups.values(); + currentActiveTaskCount = activeTaskGroups.size(); + + if (beyondProportion >= triggerSaleOutThresholdFrequency) { + // Do Scale out + int taskCount = currentActiveTaskCount + scaleOutStep; + if (currentActiveTaskCount == taskCountMax) { + log.info("CurrentActiveTaskCount reach task count Max limit, skip to scale out tasks"); + return false; + } else { + desireActiveTaskCount = Math.min(taskCount, taskCountMax); + } + log.info("Start to scale out tasks , current active task number [ " + currentActiveTaskCount + " ] and desire task number is [ " + desireActiveTaskCount + " ] "); + gracefulShutdownInternal(); + // clear everything + clearAllocationInfos(); + log.info("Set Task Count : " + desireActiveTaskCount); + setTaskCount(desireActiveTaskCount); + return true; + } + + if (withinProportion >= triggerSaleInThresholdFrequency) { + // Do Scale in + int taskCount = currentActiveTaskCount - scaleInStep; + if (currentActiveTaskCount == taskCountMin) { + log.info("CurrentActiveTaskCount reach task count Min limit, skip to scale in tasks"); + return false; + } else { + desireActiveTaskCount = Math.max(taskCount, taskCountMin); + } + log.info("Start to scale in tasks , current active task number [ " + currentActiveTaskCount + " ] and desire task number is [ " + desireActiveTaskCount + " ] "); + gracefulShutdownInternal(); + // clear everything + clearAllocationInfos(); + log.info("Set Task Count : " + desireActiveTaskCount); + setTaskCount(desireActiveTaskCount); + return true; + } + return false; + } + + private void setTaskCount(int desireActiveTaskCount) + { + ioConfig.setTaskCount(desireActiveTaskCount); + try { + Optional supervisorManager = taskMaster.getSupervisorManager(); + if (supervisorManager.isPresent()) { + MetadataSupervisorManager metadataSupervisorManager = supervisorManager.get().getMetadataSupervisorManager(); + metadataSupervisorManager.insert(dataSource, spec); + } else { + log.error("supervisorManager is null in taskMaster, skip to do scale action"); + } + } + catch (Exception e) { + log.error("Failed to sync taskCount to RDS"); + } + } + + private void clearAllocationInfos() + { + activelyReadingTaskGroups.clear(); + partitionGroups.clear(); + partitionOffsets.clear(); + + pendingCompletionTaskGroups.clear(); + partitionIds.clear(); + } + + private List collectTotalLags() + { + return queue.stream().collect(Collectors.toList()); + } + private class GracefulShutdownNotice extends ShutdownNotice { @Override @@ -469,6 +625,7 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private final SeekableStreamIndexTaskClient taskClient; private final SeekableStreamSupervisorSpec spec; private final SeekableStreamSupervisorIOConfig ioConfig; + private final Map dynamicAllocationTasksProperties; private final SeekableStreamSupervisorTuningConfig tuningConfig; private final SeekableStreamIndexTaskTuningConfig taskTuningConfig; private final String supervisorId; @@ -478,6 +635,8 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private final ExecutorService exec; private final ScheduledExecutorService scheduledExec; private final ScheduledExecutorService reportingExec; + private final ScheduledExecutorService allocationExec; + private final ScheduledExecutorService lagComputationExec; private final ListeningExecutorService workerExec; private final BlockingQueue notices = new LinkedBlockingDeque<>(); private final Object stopLock = new Object(); @@ -487,6 +646,7 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private final boolean useExclusiveStartingSequence; private boolean listenerRegistered = false; private long lastRunTime; + private long dynamicTriggerLastRunTime; private int initRetryCounter = 0; private volatile DateTime firstRunTime; private volatile DateTime earlyStopTime = null; @@ -495,6 +655,12 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private volatile boolean stopped = false; private volatile boolean lifecycleStarted = false; private final ServiceEmitter emitter; + private final boolean enableDynamicAllocationTasks; + private volatile long metricsCollectionIntervalMillis; + private volatile long metricsCollectionRangeMillis; + private volatile long dynamicCheckStartDelayMillis; + private volatile long dynamicCheckPeriod; + private volatile CircularFifoQueue queue; public SeekableStreamSupervisor( final String supervisorId, @@ -518,20 +684,52 @@ public SeekableStreamSupervisor( this.useExclusiveStartingSequence = useExclusiveStartingSequence; this.dataSource = spec.getDataSchema().getDataSource(); this.ioConfig = spec.getIoConfig(); + this.dynamicAllocationTasksProperties = ioConfig.getDynamicAllocationTasksProperties(); + log.info("Get dynamicAllocationTasksProperties from IOConfig : " + dynamicAllocationTasksProperties); + + if (dynamicAllocationTasksProperties != null && !dynamicAllocationTasksProperties.isEmpty() && Boolean.parseBoolean(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("enableDynamicAllocationTasks", false)))) { + log.info("EnableDynamicAllocationTasks for datasource " + dataSource); + this.enableDynamicAllocationTasks = true; + } else { + log.info("Disable Dynamic Allocate Tasks"); + this.enableDynamicAllocationTasks = false; + } + int taskCountMax = 0; + if (enableDynamicAllocationTasks) { + this.metricsCollectionIntervalMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionIntervalMillis", 10000))); + this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionRangeMillis", 6 * 10 * 1000))); + int slots = (int) (metricsCollectionRangeMillis / metricsCollectionIntervalMillis) + 1; + log.info(" The interval of metrics collection is " + metricsCollectionIntervalMillis + ", " + metricsCollectionRangeMillis + " timeRange will collect " + slots + " data points at most."); + this.queue = new CircularFifoQueue<>(slots); + taskCountMax = Integer.parseInt(String.valueOf(this.dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 8))); + this.dynamicCheckStartDelayMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckStartDelayMillis", 300000))); + this.dynamicCheckPeriod = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckPeriod", 600000))); + this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionRangeMillis", 600000))); + } + this.tuningConfig = spec.getTuningConfig(); this.taskTuningConfig = this.tuningConfig.convertToTaskTuningConfig(); this.supervisorId = supervisorId; this.exec = Execs.singleThreaded(supervisorId); this.scheduledExec = Execs.scheduledSingleThreaded(supervisorId + "-Scheduler-%d"); this.reportingExec = Execs.scheduledSingleThreaded(supervisorId + "-Reporting-%d"); + this.allocationExec = Execs.scheduledSingleThreaded(supervisorId + "-Allocation-%d"); + this.lagComputationExec = Execs.scheduledSingleThreaded(supervisorId + "-Computation-%d"); this.stateManager = new SeekableStreamSupervisorStateManager( spec.getSupervisorStateManagerConfig(), spec.isSuspended() ); - int workerThreads = (this.tuningConfig.getWorkerThreads() != null - ? this.tuningConfig.getWorkerThreads() - : Math.min(10, this.ioConfig.getTaskCount())); + int workerThreads = 0; + if (enableDynamicAllocationTasks) { + workerThreads = (this.tuningConfig.getWorkerThreads() != null + ? this.tuningConfig.getWorkerThreads() + : Math.min(10, taskCountMax)); + } else { + workerThreads = (this.tuningConfig.getWorkerThreads() != null + ? this.tuningConfig.getWorkerThreads() + : Math.min(10, this.ioConfig.getTaskCount())); + } this.workerExec = MoreExecutors.listeningDecorator(Execs.multiThreaded(workerThreads, supervisorId + "-Worker-%d")); log.info("Created worker pool with [%d] threads for dataSource [%s]", workerThreads, this.dataSource); @@ -571,10 +769,17 @@ public Optional getTaskStatus(String id) tuningConfig.getChatRetries() * (tuningConfig.getHttpTimeout().getStandardSeconds() + IndexTaskClient.MAX_RETRY_WAIT_SECONDS) ); + int chatThreads; + if (enableDynamicAllocationTasks) { + chatThreads = (this.tuningConfig.getChatThreads() != null + ? this.tuningConfig.getChatThreads() + : Math.min(10, taskCountMax * this.ioConfig.getReplicas())); + } else { + chatThreads = (this.tuningConfig.getChatThreads() != null + ? this.tuningConfig.getChatThreads() + : Math.min(10, this.ioConfig.getTaskCount() * this.ioConfig.getReplicas())); + } - int chatThreads = (this.tuningConfig.getChatThreads() != null - ? this.tuningConfig.getChatThreads() - : Math.min(10, this.ioConfig.getTaskCount() * this.ioConfig.getReplicas())); this.taskClient = taskClientFactory.build( taskInfoProvider, dataSource, @@ -652,6 +857,11 @@ public void stop(boolean stopGracefully) try { scheduledExec.shutdownNow(); // stop recurring executions reportingExec.shutdownNow(); + log.info("Shut Down allocationExec now"); + allocationExec.shutdownNow(); + log.info("Shut Down lagComputationExec now"); + lagComputationExec.shutdownNow(); + if (started) { Optional taskRunner = taskMaster.getTaskRunner(); @@ -768,7 +978,22 @@ public void tryInit() ); scheduleReporting(reportingExec); - + if (enableDynamicAllocationTasks) { + log.info("Collect and compute lags at fixed rate of " + metricsCollectionIntervalMillis); + lagComputationExec.scheduleAtFixedRate( + collectAndcollectLags(), + dynamicCheckStartDelayMillis, // wait for tasks to start up + metricsCollectionIntervalMillis, + TimeUnit.MILLISECONDS + ); + log.info("allocate task at fixed rate of " + dynamicCheckPeriod); + allocationExec.scheduleAtFixedRate( + buildDynamicAllocationTask(), + dynamicCheckStartDelayMillis + metricsCollectionRangeMillis, + dynamicCheckPeriod, + TimeUnit.MILLISECONDS + ); + } started = true; log.info( "Started SeekableStreamSupervisor[%s], first run in [%s], with spec: [%s]", @@ -791,6 +1016,38 @@ public void tryInit() } } + private Runnable collectAndcollectLags() + { + return new Runnable() { + @Override + public void run() + { + lock.lock(); + try { + if (!spec.isSuspended()) { + ArrayList metricsInfo = new ArrayList<>(3); + collectLag(metricsInfo); + long totalLags = metricsInfo.size() < 3 ? 0 : metricsInfo.get(1); + queue.offer(totalLags > 0 ? totalLags : 0); + log.info("Current lag metric points : " + queue.stream().collect(Collectors.toList())); + } else { + log.info("[%s] supervisor is suspended, skip to collect kafka lags", dataSource); + } + } + catch (Exception e) { + log.error(e, "Error, When collect kafka lags"); + } + finally { + lock.unlock(); + } + } + }; + } + private Runnable buildDynamicAllocationTask() + { + return () -> notices.add(new DynamicAllocationTasksNotice()); + } + private Runnable buildRunTask() { return () -> notices.add(new RunNotice()); @@ -1137,6 +1394,20 @@ public void gracefulShutdownInternal() throws ExecutionException, InterruptedExc @VisibleForTesting public void resetInternal(DataSourceMetadata dataSourceMetadata) { + // clear queue for kafka lags + if (enableDynamicAllocationTasks && queue != null) { + try { + lock.lock(); + queue.clear(); + } + catch (Exception e) { + log.warn(e, "Error,when clear queue in rest action"); + } + finally { + lock.unlock(); + } + } + if (dataSourceMetadata == null) { // Reset everything boolean result = indexerMetadataStorageCoordinator.deleteDataSourceMetadata(dataSource); @@ -3487,33 +3758,25 @@ protected void emitLag() final String type = spec.getType(); BiConsumer, String> emitFn = (partitionLags, suffix) -> { - if (partitionLags == null) { + ArrayList lags = new ArrayList<>(3); + computeLags(partitionLags, lags); + if (lags.size() < 3) { return; } - - long maxLag = 0, totalLag = 0, avgLag; - for (long lag : partitionLags.values()) { - if (lag > maxLag) { - maxLag = lag; - } - totalLag += lag; - } - avgLag = partitionLags.size() == 0 ? 0 : totalLag / partitionLags.size(); - emitter.emit( ServiceMetricEvent.builder() .setDimension("dataSource", dataSource) - .build(StringUtils.format("ingest/%s/lag%s", type, suffix), totalLag) + .build(StringUtils.format("ingest/%s/lag%s", type, suffix), lags.get(1)) ); emitter.emit( ServiceMetricEvent.builder() .setDimension("dataSource", dataSource) - .build(StringUtils.format("ingest/%s/maxLag%s", type, suffix), maxLag) + .build(StringUtils.format("ingest/%s/maxLag%s", type, suffix), lags.get(0)) ); emitter.emit( ServiceMetricEvent.builder() .setDimension("dataSource", dataSource) - .build(StringUtils.format("ingest/%s/avgLag%s", type, suffix), avgLag) + .build(StringUtils.format("ingest/%s/avgLag%s", type, suffix), lags.get(2)) ); }; @@ -3526,6 +3789,25 @@ protected void emitLag() } } + + protected void computeLags(Map partitionLags, ArrayList lags) + { + if (partitionLags == null) { + return; + } + long maxLag = 0, totalLag = 0, avgLag; + for (long lag : partitionLags.values()) { + if (lag > maxLag) { + maxLag = lag; + } + totalLag += lag; + } + avgLag = partitionLags.size() == 0 ? 0 : totalLag / partitionLags.size(); + lags.add(maxLag); + lags.add(totalLag); + lags.add(avgLag); + } + /** * a special sequence number that is used to indicate that the sequence offset * for a particular partition has not yet been calculated by the supervisor. When @@ -3561,4 +3843,6 @@ protected void emitLag() * sequences. In Kafka, start offsets are always inclusive. */ protected abstract boolean useExclusiveStartSequenceNumberForNonFirstSequence(); + + protected abstract void collectLag(ArrayList lags); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java index 723e22ec5183..31dc360f1705 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java @@ -30,6 +30,8 @@ import javax.annotation.Nullable; +import java.util.Map; + public abstract class SeekableStreamSupervisorIOConfig { @@ -37,7 +39,7 @@ public abstract class SeekableStreamSupervisorIOConfig @Nullable private final InputFormat inputFormat; // nullable for backward compatibility private final Integer replicas; - private final Integer taskCount; + private Integer taskCount; private final Duration taskDuration; private final Duration startDelay; private final Duration period; @@ -46,6 +48,7 @@ public abstract class SeekableStreamSupervisorIOConfig private final Optional lateMessageRejectionPeriod; private final Optional earlyMessageRejectionPeriod; private final Optional lateMessageRejectionStartDateTime; + private final Map dynamicAllocationTasksProperties; public SeekableStreamSupervisorIOConfig( String stream, @@ -59,6 +62,7 @@ public SeekableStreamSupervisorIOConfig( Period completionTimeout, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, + Map dynamicAllocationTasksProperties, DateTime lateMessageRejectionStartDateTime ) { @@ -87,6 +91,8 @@ public SeekableStreamSupervisorIOConfig( + "both properties lateMessageRejectionStartDateTime " + "and lateMessageRejectionPeriod."); } + // Could be null + this.dynamicAllocationTasksProperties = dynamicAllocationTasksProperties; } private static Duration defaultDuration(final Period period, final String theDefault) @@ -113,12 +119,23 @@ public Integer getReplicas() return replicas; } + @JsonProperty + public Map getDynamicAllocationTasksProperties() + { + return dynamicAllocationTasksProperties; + } + @JsonProperty public Integer getTaskCount() { return taskCount; } + public void setTaskCount(final int taskCount) + { + this.taskCount = taskCount; + } + @JsonProperty public Duration getTaskDuration() { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index 6bfbd7f9983f..c21eeeee22e7 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -764,6 +764,7 @@ private void expectEmitterSupervisor(boolean suspended) throws EntryExistsExcept new Period("PT30M"), null, null, + null, null ) { @@ -824,7 +825,7 @@ private static SeekableStreamSupervisorIOConfig getIOConfig() false, new Period("PT30M"), null, - null, null + null, null, null ) { }; @@ -1174,6 +1175,11 @@ protected void scheduleReporting(ScheduledExecutorService reportingExec) { // do nothing } + + @Override + protected void collectLag(ArrayList lags) + { + } } private class TestEmittingTestSeekableStreamSupervisor extends BaseTestSeekableStreamSupervisor @@ -1216,6 +1222,11 @@ protected void emitLag() } } + @Override + protected void collectLag(ArrayList lags) + { + } + @Override protected void scheduleReporting(ScheduledExecutorService reportingExec) { From 16b07446ad051fd8277bdba14b9b3bde4bab247b Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 21 Oct 2020 11:25:00 +0800 Subject: [PATCH 04/47] fix kafkaSupervisorIOConfig and KinesisSupervisorIOConfig --- .idea/misc.xml | 4 ++-- .../indexing/kafka/supervisor/KafkaSupervisorIOConfig.java | 2 ++ .../apache/druid/indexing/kafka/KafkaSamplerSpecTest.java | 1 + .../druid/indexing/kafka/supervisor/KafkaSupervisorTest.java | 3 +++ .../kinesis/supervisor/KinesisSupervisorIOConfig.java | 5 +++++ .../druid/indexing/kinesis/KinesisSamplerSpecTest.java | 1 + .../indexing/kinesis/supervisor/KinesisSupervisorTest.java | 5 +++++ 7 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java index a1360b5c63f3..a2653d2ce025 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java @@ -50,6 +50,7 @@ public KafkaSupervisorIOConfig( @JsonProperty("taskCount") Integer taskCount, @JsonProperty("taskDuration") Period taskDuration, @JsonProperty("consumerProperties") Map consumerProperties, + @JsonProperty("dynamicAllocationTasksProperties") Map dynamicAllocationTasksProperties, @JsonProperty("pollTimeout") Long pollTimeout, @JsonProperty("startDelay") Period startDelay, @JsonProperty("period") Period period, @@ -72,6 +73,7 @@ public KafkaSupervisorIOConfig( completionTimeout, lateMessageRejectionPeriod, earlyMessageRejectionPeriod, + dynamicAllocationTasksProperties, lateMessageRejectionStartDateTime ); diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java index 3768cebd62da..22553dec571d 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaSamplerSpecTest.java @@ -134,6 +134,7 @@ public void testSample() null, null, null, + null, true, null, null, diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 905abd19e2ac..be402652b8a2 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -3372,6 +3372,7 @@ private TestableKafkaSupervisor getTestableSupervisor( taskCount, new Period(duration), consumerProperties, + null, KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), @@ -3482,6 +3483,7 @@ private TestableKafkaSupervisor getTestableSupervisorCustomIsTaskCurrent( taskCount, new Period(duration), consumerProperties, + null, KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), @@ -3596,6 +3598,7 @@ private KafkaSupervisor getSupervisor( taskCount, new Period(duration), consumerProperties, + null, KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java index f68e0b7a09b3..885fb6e9f7dd 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java @@ -29,6 +29,8 @@ import org.joda.time.DateTime; import org.joda.time.Period; +import java.util.Map; + public class KinesisSupervisorIOConfig extends SeekableStreamSupervisorIOConfig { private final String endpoint; @@ -70,6 +72,7 @@ public KinesisSupervisorIOConfig( @JsonProperty("fetchDelayMillis") Integer fetchDelayMillis, @JsonProperty("awsAssumedRoleArn") String awsAssumedRoleArn, @JsonProperty("awsExternalId") String awsExternalId, + @JsonProperty("dynamicAllocationTasksProperties") Map dynamicAllocationTasksProperties, @JsonProperty("deaggregate") boolean deaggregate ) { @@ -85,7 +88,9 @@ public KinesisSupervisorIOConfig( completionTimeout, lateMessageRejectionPeriod, earlyMessageRejectionPeriod, + dynamicAllocationTasksProperties, lateMessageRejectionStartDateTime + ); this.endpoint = endpoint != null ? endpoint diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java index dadc6c7a944d..0042b0278f84 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisSamplerSpecTest.java @@ -152,6 +152,7 @@ public void testSample() throws Exception null, null, null, + null, false ), null, diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java index 135166250d99..47871e71fa80 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java @@ -301,6 +301,7 @@ public void testRecordSupplier() 1000, null, null, + null, false ); KinesisIndexTaskClientFactory clientFactory = new KinesisIndexTaskClientFactory(null, OBJECT_MAPPER); @@ -4716,6 +4717,7 @@ private TestableKinesisSupervisor getTestableSupervisor( null, null, null, + null, false ); @@ -4856,6 +4858,7 @@ private TestableKinesisSupervisor getTestableSupervisor( fetchDelayMillis, null, null, + null, false ); @@ -4943,6 +4946,7 @@ private TestableKinesisSupervisor getTestableSupervisorCustomIsTaskCurrent( fetchDelayMillis, null, null, + null, false ); @@ -5032,6 +5036,7 @@ private KinesisSupervisor getSupervisor( fetchDelayMillis, null, null, + null, false ); From 07eb9c089b9ba751dcb4c3b55728c3486406b278 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 21 Oct 2020 11:28:36 +0800 Subject: [PATCH 05/47] test dynamic auto scale done --- .idea/misc.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..ff138cc554d4 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + \ No newline at end of file From 746b033327ef51dd74c811c369444e51bac785ca Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 21 Oct 2020 11:32:06 +0800 Subject: [PATCH 06/47] auto scale tasks tested on prd cluster --- .idea/misc.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index ff138cc554d4..61e7e5b21e77 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + \ No newline at end of file From d25f94a593a7b47207db44a8df838bbea46d0793 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 21 Oct 2020 11:33:04 +0800 Subject: [PATCH 07/47] auto scale tasks tested on prd cluster --- .idea/misc.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index 61e7e5b21e77..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -87,4 +87,4 @@ - \ No newline at end of file + From e7a1af1e97a8f693b8f43ef6a20cda5a735e7fa4 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 22 Oct 2020 18:56:00 +0800 Subject: [PATCH 08/47] modify code style to solve 29055.10 29055.9 29055.17 29055.18 29055.19 29055.20 --- indexing-service/pom.xml | 5 ++++ .../supervisor/SeekableStreamSupervisor.java | 8 +++---- .../SeekableStreamSupervisorStateTest.java | 24 ++++++++++++++++++- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index d00de06aa8d5..666e4e58b1de 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -67,6 +67,11 @@ io.dropwizard.metrics metrics-core + + org.apache.commons + commons-collections4 + 4.2 + com.google.code.findbugs jsr305 diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 4b3e8a364040..16abe47e2068 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -398,7 +398,7 @@ private boolean dynamicAllocate(List lags) throws InterruptedException, Ex log.info("triggerSaleOutThresholdFrequency is [ " + triggerSaleOutThresholdFrequency + " ] and triggerSaleInThresholdFrequency is [ " + triggerSaleInThresholdFrequency + " ]"); log.info("beyondProportion is [ " + beyondProportion + " ] and withinProportion is [ " + withinProportion + " ]"); - int currentActiveTaskCount = 0; + int currentActiveTaskCount; int desireActiveTaskCount; Collection activeTaskGroups = activelyReadingTaskGroups.values(); currentActiveTaskCount = activeTaskGroups.size(); @@ -470,7 +470,7 @@ private void clearAllocationInfos() private List collectTotalLags() { - return queue.stream().collect(Collectors.toList()); + return new ArrayList<>(queue); } private class GracefulShutdownNotice extends ShutdownNotice @@ -720,7 +720,7 @@ public SeekableStreamSupervisor( spec.isSuspended() ); - int workerThreads = 0; + int workerThreads; if (enableDynamicAllocationTasks) { workerThreads = (this.tuningConfig.getWorkerThreads() != null ? this.tuningConfig.getWorkerThreads() @@ -1029,7 +1029,7 @@ public void run() collectLag(metricsInfo); long totalLags = metricsInfo.size() < 3 ? 0 : metricsInfo.get(1); queue.offer(totalLags > 0 ? totalLags : 0); - log.info("Current lag metric points : " + queue.stream().collect(Collectors.toList())); + log.info("Current lag metric points : " + new ArrayList<>(queue)); } else { log.info("[%s] supervisor is suspended, skip to collect kafka lags", dataSource); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index c21eeeee22e7..d644088b89f9 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -81,9 +81,11 @@ import org.junit.Test; import javax.annotation.Nullable; + import java.io.File; import java.math.BigInteger; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -825,12 +827,32 @@ private static SeekableStreamSupervisorIOConfig getIOConfig() false, new Period("PT30M"), null, - null, null, null + null, getDynamicAllocationTasksProperties(), null ) { }; } + private static Map getDynamicAllocationTasksProperties() + { + HashMap dynamicAllocationTasksProperties = new HashMap<>(); + dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", true); + dynamicAllocationTasksProperties.put("metricsCollectionIntervalMillis", 500); + dynamicAllocationTasksProperties.put("metricsCollectionRangeMillis", 500); + dynamicAllocationTasksProperties.put("scaleOutThreshold", 5000000); + dynamicAllocationTasksProperties.put("triggerSaleOutThresholdFrequency", 0.3); + dynamicAllocationTasksProperties.put("scaleInThreshold", 1000000); + dynamicAllocationTasksProperties.put("triggerSaleInThresholdFrequency", 0.8); + dynamicAllocationTasksProperties.put("dynamicCheckStartDelayMillis", 0); + dynamicAllocationTasksProperties.put("dynamicCheckPeriod", 100); + dynamicAllocationTasksProperties.put("taskCountMax", 8); + dynamicAllocationTasksProperties.put("taskCountMin", 1); + dynamicAllocationTasksProperties.put("scaleInStep", 1); + dynamicAllocationTasksProperties.put("scaleOutStep", 2); + dynamicAllocationTasksProperties.put("minTriggerDynamicFrequencyMillis", 1200000); + return dynamicAllocationTasksProperties; + } + private static SeekableStreamSupervisorTuningConfig getTuningConfig() { return new SeekableStreamSupervisorTuningConfig() From 78cbd45577dccc3abd39fb03db6d2a9298e6c252 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 23 Oct 2020 12:54:40 +0800 Subject: [PATCH 09/47] rename test fiel function --- .../supervisor/SeekableStreamSupervisorStateTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index d644088b89f9..e3d82c6d94d8 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -827,13 +827,13 @@ private static SeekableStreamSupervisorIOConfig getIOConfig() false, new Period("PT30M"), null, - null, getDynamicAllocationTasksProperties(), null + null, getProperties(), null ) { }; } - private static Map getDynamicAllocationTasksProperties() + private static Map getProperties() { HashMap dynamicAllocationTasksProperties = new HashMap<>(); dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", true); From 215844e6162e7095450b04aa6218989da939f37a Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 27 Nov 2020 20:07:24 +0800 Subject: [PATCH 10/47] change codes and add docs based on capistrant reviewed --- .../extensions-core/kafka-ingestion.md | 20 ++ .../kafka/supervisor/KafkaSupervisor.java | 6 +- .../kinesis/supervisor/KinesisSupervisor.java | 1 + .../supervisor/KinesisSupervisorIOConfig.java | 1 - .../supervisor/SeekableStreamSupervisor.java | 174 +++++++++++------- .../SeekableStreamSupervisorIOConfig.java | 5 +- 6 files changed, 134 insertions(+), 73 deletions(-) diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 3eb3499fb9e7..3aed574a53a7 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -144,6 +144,26 @@ A sample supervisor spec is shown below: |`lateMessageRejectionStartDateTime`|ISO8601 DateTime|Configure tasks to reject messages with timestamps earlier than this date time; for example if this is set to `2016-01-01T11:00Z` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline).|no (default == none)| |`lateMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps earlier than this period before the task was created; for example if this is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline). Please note that only one of `lateMessageRejectionPeriod` or `lateMessageRejectionStartDateTime` can be specified.|no (default == none)| |`earlyMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps later than this period after the task reached its taskDuration; for example if this is set to `PT1H`, the taskDuration is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps later than *2016-01-01T14:00Z* will be dropped. **Note:** Tasks sometimes run past their task duration, for example, in cases of supervisor failover. Setting earlyMessageRejectionPeriod too low may cause messages to be dropped unexpectedly whenever a task runs past its originally configured task duration.|no (default == none)| +|`dynamicAllocationTasksProperties`|Object|`dynamicAllocationTasksProperties` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. See below `Dynamic Allocation Tasks Properties` for details.|no (default == null)| + +#### Dynamic Allocation Tasks Properties + +| Property | Description | Default | +| ------------- | ------------- | ------------- | +| enableDynamicAllocationTasks | whether enable this feature or not | false | +| metricsCollectionIntervalMillis | Define the frequency of lag points collection. | 30000 | +| metricsCollectionRangeMillis | The total time window of lag collection, Use with metricsCollectionIntervalMillis，it means that in the recent metricsCollectionRangeMill, collect lag metric points every metricsCollectionIntervalMillis. | 600000 | +| scaleOutThreshold | The Threshold of scale out action | 6000000 | +| triggerScaleOutThresholdFrequency | If 'triggerScaleOutThresholdFrequency' percent of lag points are higher than scaleOutThreshold, then do scale out action. | 0.3 | +| scaleInThreshold | The Threshold of scale in action | 1000000 | +| triggerScaleInThresholdFrequency | If 'triggerScaleInThresholdFrequency' percent of lag points are lower than scaleOutThreshold, then do scale in action. | 0.9 | +| dynamicCheckStartDelayMillis | Number of milliseconds after supervisor starts when first check scale logic. | 300000 | +| dynamicCheckPeriod | the frequency of checking wheather to do scale action | 60000 | +| taskCountMax | Maximum value of task count | 4 | +| taskCountMin | Minimum value of task count | 1 | +| scaleInStep | How many tasks to reduce at once | 1 | +| scaleOutStep | How many tasks to add at a time | 2 | +| minTriggerDynamicFrequencyMillis | Minimum time interval between two scale actions | 600000 | #### Specifying data format diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java index 8f87e023c876..d245cd15512a 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java @@ -333,7 +333,11 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() @Override protected void collectLag(ArrayList lags) { - computeLags(getPartitionRecordLag(), lags); + Map partitionRecordLag = getPartitionRecordLag(); + if (partitionRecordLag == null) { + return; + } + computeLags(partitionRecordLag, lags); } @Override diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java index a1f17c5da512..28272d41eb2e 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java @@ -377,6 +377,7 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() return true; } + // not yet supported, will be implemented in the future maybe? need to find a proper way to measure kinesis lag. @Override protected void collectLag(ArrayList lags) { diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java index 885fb6e9f7dd..131698186586 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java @@ -90,7 +90,6 @@ public KinesisSupervisorIOConfig( earlyMessageRejectionPeriod, dynamicAllocationTasksProperties, lateMessageRejectionStartDateTime - ); this.endpoint = endpoint != null ? endpoint diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 16abe47e2068..22554b741592 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -322,46 +322,47 @@ public void handle() } } - // same as submit supervisor logic + // change taskCount without resubmitting. private class DynamicAllocationTasksNotice implements Notice { + /** + * This method will do lags points collection and check dynamic scale action is necessary or not. + */ @Override public void handle() { lock.lock(); try { long nowTime = System.currentTimeMillis(); - long minTriggerDynamicFrequency = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("minTriggerDynamicFrequencyMillis", 1200000))); // Only queue is full and over minTriggerDynamicFrequency can trigger scale out/in - // max(minTriggerDynamicFrequency, metricsCollectionRangeMillis) if (spec.isSuspended()) { log.info("[%s] supervisor is suspended, skip to check dynamic allocate task logic", dataSource); return; } - log.info("PendingCompletionTaskGroups is : " + pendingCompletionTaskGroups); + log.debug("PendingCompletionTaskGroups is [%s] for dataSource [%s].", pendingCompletionTaskGroups, dataSource); for (CopyOnWriteArrayList list : pendingCompletionTaskGroups.values()) { if (!list.isEmpty()) { - log.info("Still hand off tasks unfinished, skip to do scale action [" + pendingCompletionTaskGroups + "]"); + log.info("Still hand off tasks unfinished, skip to do scale action [%s] for dataSource [%s].", pendingCompletionTaskGroups, dataSource); return; } } if (nowTime - dynamicTriggerLastRunTime < minTriggerDynamicFrequency) { - log.info("NowTime - dynamicTriggerLastRunTime is [" + (nowTime - dynamicTriggerLastRunTime) + "]. Defined minTriggerDynamicFrequency is [" + minTriggerDynamicFrequency + "] , CLAM DOWN NOW !"); + log.info("NowTime - dynamicTriggerLastRunTime is [%s]. Defined minTriggerDynamicFrequency is [%s] for dataSource [%s], CLAM DOWN NOW !", nowTime - dynamicTriggerLastRunTime, minTriggerDynamicFrequency, dataSource); return; } - if (!queue.isAtFullCapacity()) { - log.info("Metrics collection is not at full capacity, skip to check dynamic allocate task : [" + queue.size() + " vs " + queue.maxSize() + "]"); + if (!lagMetricsQueue.isAtFullCapacity()) { + log.info("Metrics collection is not at full capacity, may cause unnecessary scale. Skip to check dynamic allocate task : [%s] vs [%s]", lagMetricsQueue.size(), lagMetricsQueue.maxSize()); return; } List lags = collectTotalLags(); boolean allocationSuccess = dynamicAllocate(lags); if (allocationSuccess) { dynamicTriggerLastRunTime = nowTime; - queue.clear(); + lagMetricsQueue.clear(); } } catch (Exception e) { - log.error(e, "Error, when parse DynamicAllocationTasksNotice"); + log.warn(e, "Error, when parse DynamicAllocationTasksNotice"); } finally { lock.unlock(); @@ -369,19 +370,29 @@ public void handle() } } + /** + * This method determines whether and how to do scale actions based on collected lag points. + * Current algorithm of scale is simple: + * First of all, compute the proportion of lag points higher/lower than scaleOutThreshold/scaleInThreshold, getting scaleOutThreshold/scaleInThreshold. + * Secondly, compare scaleOutThreshold/scaleInThreshold with triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency. P.S. Scale out action has higher priority than scale in action. + * Finaly, if scaleOutThreshold/scaleInThreshold is higher than triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency, scale out/in action would be triggered. + * If scale action is triggered : + * First of all, call gracefulShutdownInternal() which will change the state of current datasource ingest tasks from reading to publishing. + * Secondly, clear all the stateful data structures: activelyReadingTaskGroups, partitionGroups, partitionOffsets, pendingCompletionTaskGroups, partitionIds. These structures will be rebuiled next 'RunNotice'. + * Finally, change taskCount in SeekableStreamSupervisorIOConfig and sync it to MetaStorage. + * After changed taskCount in SeekableStreamSupervisorIOConfig, next RunNotice will ceate scaled number of ingest tasks without resubmitting supervisors. + * @param lags the lag metrics of Stream(Kafka/Kinesis) + * @return Boolean flag, do scale action successfully or not. If true , it will take at least 'minTriggerDynamicFrequency' before next 'dynamicAllocatie'. + * If false, it will do 'dynamicAllocate' again after 'dynamicCheckPeriod'. + * @throws InterruptedException + * @throws ExecutionException + * @throws TimeoutException + */ private boolean dynamicAllocate(List lags) throws InterruptedException, ExecutionException, TimeoutException { // if supervisor is not suspended, ensure required tasks are running // if suspended, ensure tasks have been requested to gracefully stop - log.info("[%s] supervisor is running, start to check dynamic allocate task logic", dataSource); - long scaleOutThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutThreshold", 5000000))); - long scaleInThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInThreshold", 1000000))); - double triggerSaleOutThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerSaleOutThresholdFrequency", 0.3))); - double triggerSaleInThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerSaleInThresholdFrequency", 0.8))); - int taskCountMax = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 8))); - int taskCountMin = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMin", 1))); - int scaleInStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInStep", 1))); - int scaleOutStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutStep", 2))); + log.info("[%s] supervisor is running, start to check dynamic allocate task logic. Current collected lags : [%s]", dataSource, lags); int beyond = 0; int within = 0; int metricsCount = lags.size(); @@ -395,53 +406,53 @@ private boolean dynamicAllocate(List lags) throws InterruptedException, Ex } double beyondProportion = beyond * 1.0 / metricsCount; double withinProportion = within * 1.0 / metricsCount; - log.info("triggerSaleOutThresholdFrequency is [ " + triggerSaleOutThresholdFrequency + " ] and triggerSaleInThresholdFrequency is [ " + triggerSaleInThresholdFrequency + " ]"); - log.info("beyondProportion is [ " + beyondProportion + " ] and withinProportion is [ " + withinProportion + " ]"); + log.debug("triggerScaleOutThresholdFrequency is [%s] and triggerScaleInThresholdFrequency is [%s] for dataSource [%s].", triggerScaleOutThresholdFrequency, triggerScaleInThresholdFrequency, dataSource); + log.info("beyondProportion is [%s] and withinProportion is [%s] for dataSource [%s].", beyondProportion, withinProportion, dataSource); int currentActiveTaskCount; int desireActiveTaskCount; Collection activeTaskGroups = activelyReadingTaskGroups.values(); currentActiveTaskCount = activeTaskGroups.size(); - if (beyondProportion >= triggerSaleOutThresholdFrequency) { + if (beyondProportion >= triggerScaleOutThresholdFrequency) { // Do Scale out int taskCount = currentActiveTaskCount + scaleOutStep; if (currentActiveTaskCount == taskCountMax) { - log.info("CurrentActiveTaskCount reach task count Max limit, skip to scale out tasks"); + log.info("CurrentActiveTaskCount reach task count Max limit, skip to scale out tasks for dataSource [%s].", dataSource); return false; } else { desireActiveTaskCount = Math.min(taskCount, taskCountMax); } - log.info("Start to scale out tasks , current active task number [ " + currentActiveTaskCount + " ] and desire task number is [ " + desireActiveTaskCount + " ] "); + log.debug("Start to scale out tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desireActiveTaskCount, dataSource); gracefulShutdownInternal(); // clear everything clearAllocationInfos(); - log.info("Set Task Count : " + desireActiveTaskCount); - setTaskCount(desireActiveTaskCount); + log.info("Change taskCount to [%s] for dataSource [%s].", desireActiveTaskCount, dataSource); + changeTaskCountInIOConfig(desireActiveTaskCount); return true; } - if (withinProportion >= triggerSaleInThresholdFrequency) { + if (withinProportion >= triggerScaleInThresholdFrequency) { // Do Scale in int taskCount = currentActiveTaskCount - scaleInStep; if (currentActiveTaskCount == taskCountMin) { - log.info("CurrentActiveTaskCount reach task count Min limit, skip to scale in tasks"); + log.info("CurrentActiveTaskCount reach task count Min limit, skip to scale in tasks for dataSource [%s].", dataSource); return false; } else { desireActiveTaskCount = Math.max(taskCount, taskCountMin); } - log.info("Start to scale in tasks , current active task number [ " + currentActiveTaskCount + " ] and desire task number is [ " + desireActiveTaskCount + " ] "); + log.debug("Start to scale in tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desireActiveTaskCount, dataSource); gracefulShutdownInternal(); // clear everything clearAllocationInfos(); - log.info("Set Task Count : " + desireActiveTaskCount); - setTaskCount(desireActiveTaskCount); + log.info("Change taskCount to [%s] for dataSource [%s].", desireActiveTaskCount, dataSource); + changeTaskCountInIOConfig(desireActiveTaskCount); return true; } return false; } - private void setTaskCount(int desireActiveTaskCount) + private void changeTaskCountInIOConfig(int desireActiveTaskCount) { ioConfig.setTaskCount(desireActiveTaskCount); try { @@ -450,11 +461,11 @@ private void setTaskCount(int desireActiveTaskCount) MetadataSupervisorManager metadataSupervisorManager = supervisorManager.get().getMetadataSupervisorManager(); metadataSupervisorManager.insert(dataSource, spec); } else { - log.error("supervisorManager is null in taskMaster, skip to do scale action"); + log.warn("supervisorManager is null in taskMaster, skip to do scale action for dataSource [%s].", dataSource); } } catch (Exception e) { - log.error("Failed to sync taskCount to RDS"); + log.warn("Failed to sync taskCount to MetaStorage for dataSource [%s].", dataSource); } } @@ -470,7 +481,7 @@ private void clearAllocationInfos() private List collectTotalLags() { - return new ArrayList<>(queue); + return new ArrayList<>(lagMetricsQueue); } private class GracefulShutdownNotice extends ShutdownNotice @@ -656,11 +667,21 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private volatile boolean lifecycleStarted = false; private final ServiceEmitter emitter; private final boolean enableDynamicAllocationTasks; - private volatile long metricsCollectionIntervalMillis; - private volatile long metricsCollectionRangeMillis; - private volatile long dynamicCheckStartDelayMillis; - private volatile long dynamicCheckPeriod; - private volatile CircularFifoQueue queue; + private long metricsCollectionIntervalMillis; + private long metricsCollectionRangeMillis; + private long scaleOutThreshold; + private double triggerScaleOutThresholdFrequency; + private long scaleInThreshold; + private double triggerScaleInThresholdFrequency; + private long dynamicCheckStartDelayMillis; + private long dynamicCheckPeriod; + private int taskCountMax; + private int taskCountMin; + private int scaleInStep; + private int scaleOutStep; + private long minTriggerDynamicFrequency; + + private volatile CircularFifoQueue lagMetricsQueue; public SeekableStreamSupervisor( final String supervisorId, @@ -685,26 +706,32 @@ public SeekableStreamSupervisor( this.dataSource = spec.getDataSchema().getDataSource(); this.ioConfig = spec.getIoConfig(); this.dynamicAllocationTasksProperties = ioConfig.getDynamicAllocationTasksProperties(); - log.info("Get dynamicAllocationTasksProperties from IOConfig : " + dynamicAllocationTasksProperties); - + log.debug("Get dynamicAllocationTasksProperties from IOConfig : [%s] in [%s]", dynamicAllocationTasksProperties, dataSource); if (dynamicAllocationTasksProperties != null && !dynamicAllocationTasksProperties.isEmpty() && Boolean.parseBoolean(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("enableDynamicAllocationTasks", false)))) { - log.info("EnableDynamicAllocationTasks for datasource " + dataSource); + log.info("EnableDynamicAllocationTasks for datasource [%s]", dataSource); this.enableDynamicAllocationTasks = true; } else { - log.info("Disable Dynamic Allocate Tasks"); + log.info("Disable dynamic allocate tasks for [%s]", dataSource); this.enableDynamicAllocationTasks = false; } - int taskCountMax = 0; if (enableDynamicAllocationTasks) { - this.metricsCollectionIntervalMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionIntervalMillis", 10000))); - this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionRangeMillis", 6 * 10 * 1000))); + this.metricsCollectionIntervalMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionIntervalMillis", 30000))); + this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionRangeMillis", 600000))); int slots = (int) (metricsCollectionRangeMillis / metricsCollectionIntervalMillis) + 1; - log.info(" The interval of metrics collection is " + metricsCollectionIntervalMillis + ", " + metricsCollectionRangeMillis + " timeRange will collect " + slots + " data points at most."); - this.queue = new CircularFifoQueue<>(slots); - taskCountMax = Integer.parseInt(String.valueOf(this.dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 8))); + log.debug(" The interval of metrics collection is [%s], [%s] timeRange will collect [%s] data points for dataSource [%s].", metricsCollectionIntervalMillis, metricsCollectionRangeMillis, slots, dataSource); + this.lagMetricsQueue = new CircularFifoQueue<>(slots); this.dynamicCheckStartDelayMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckStartDelayMillis", 300000))); - this.dynamicCheckPeriod = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckPeriod", 600000))); + this.dynamicCheckPeriod = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckPeriod", 60000))); this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionRangeMillis", 600000))); + this.scaleOutThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutThreshold", 6000000))); + this.scaleInThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInThreshold", 1000000))); + this.triggerScaleOutThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerScaleOutThresholdFrequency", 0.3))); + this.triggerScaleInThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerScaleInThresholdFrequency", 0.9))); + this.taskCountMax = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 4))); + this.taskCountMin = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMin", 1))); + this.scaleInStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInStep", 1))); + this.scaleOutStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutStep", 2))); + this.minTriggerDynamicFrequency = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("minTriggerDynamicFrequencyMillis", 600000))); } this.tuningConfig = spec.getTuningConfig(); @@ -857,9 +884,7 @@ public void stop(boolean stopGracefully) try { scheduledExec.shutdownNow(); // stop recurring executions reportingExec.shutdownNow(); - log.info("Shut Down allocationExec now"); allocationExec.shutdownNow(); - log.info("Shut Down lagComputationExec now"); lagComputationExec.shutdownNow(); @@ -979,14 +1004,14 @@ public void tryInit() scheduleReporting(reportingExec); if (enableDynamicAllocationTasks) { - log.info("Collect and compute lags at fixed rate of " + metricsCollectionIntervalMillis); + log.debug("Collect and compute lags at fixed rate of [%s] for dataSource[%s].", metricsCollectionIntervalMillis, dataSource); lagComputationExec.scheduleAtFixedRate( - collectAndcollectLags(), + collectAndComputeLags(), dynamicCheckStartDelayMillis, // wait for tasks to start up metricsCollectionIntervalMillis, TimeUnit.MILLISECONDS ); - log.info("allocate task at fixed rate of " + dynamicCheckPeriod); + log.debug("allocate task at fixed rate of [%s], dataSource [%s].", dynamicCheckPeriod, dataSource); allocationExec.scheduleAtFixedRate( buildDynamicAllocationTask(), dynamicCheckStartDelayMillis + metricsCollectionRangeMillis, @@ -1016,7 +1041,11 @@ public void tryInit() } } - private Runnable collectAndcollectLags() + /** + * This method compute current consume lags. Get the total lags of all partition and fill in lagMetricsQueue + * @return a Runnbale object to do collect and compute action. + */ + private Runnable collectAndComputeLags() { return new Runnable() { @Override @@ -1028,14 +1057,14 @@ public void run() ArrayList metricsInfo = new ArrayList<>(3); collectLag(metricsInfo); long totalLags = metricsInfo.size() < 3 ? 0 : metricsInfo.get(1); - queue.offer(totalLags > 0 ? totalLags : 0); - log.info("Current lag metric points : " + new ArrayList<>(queue)); + lagMetricsQueue.offer(totalLags > 0 ? totalLags : 0); + log.debug("Current lag metric points [%s] for dataSource [%s].", new ArrayList<>(lagMetricsQueue), dataSource); } else { - log.info("[%s] supervisor is suspended, skip to collect kafka lags", dataSource); + log.debug("[%s] supervisor is suspended, skip to collect kafka lags", dataSource); } } catch (Exception e) { - log.error(e, "Error, When collect kafka lags"); + log.warn(e, "Error, When collect kafka lags"); } finally { lock.unlock(); @@ -1395,10 +1424,10 @@ public void gracefulShutdownInternal() throws ExecutionException, InterruptedExc public void resetInternal(DataSourceMetadata dataSourceMetadata) { // clear queue for kafka lags - if (enableDynamicAllocationTasks && queue != null) { + if (enableDynamicAllocationTasks && lagMetricsQueue != null) { try { lock.lock(); - queue.clear(); + lagMetricsQueue.clear(); } catch (Exception e) { log.warn(e, "Error,when clear queue in rest action"); @@ -3758,11 +3787,11 @@ protected void emitLag() final String type = spec.getType(); BiConsumer, String> emitFn = (partitionLags, suffix) -> { - ArrayList lags = new ArrayList<>(3); - computeLags(partitionLags, lags); - if (lags.size() < 3) { + if (partitionLags == null) { return; } + ArrayList lags = new ArrayList<>(3); + computeLags(partitionLags, lags); emitter.emit( ServiceMetricEvent.builder() .setDimension("dataSource", dataSource) @@ -3790,11 +3819,13 @@ protected void emitLag() } + /** + * This method compute maxLag, totalLag and avgLag then fill in 'lags' + * @param partitionLags lags per partition + * @param lags a arraylist in order of maxLag, totalLag and avgLag + */ protected void computeLags(Map partitionLags, ArrayList lags) { - if (partitionLags == null) { - return; - } long maxLag = 0, totalLag = 0, avgLag; for (long lag : partitionLags.values()) { if (lag > maxLag) { @@ -3844,5 +3875,10 @@ protected void computeLags(Map partitionLags, ArrayList lags + * Only support Kafka ingestion so far. + * @param lags , Notice : The order of values is maxLag, totalLag and avgLag. + */ protected abstract void collectLag(ArrayList lags); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java index 31dc360f1705..80671c5eefe5 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java @@ -48,7 +48,7 @@ public abstract class SeekableStreamSupervisorIOConfig private final Optional lateMessageRejectionPeriod; private final Optional earlyMessageRejectionPeriod; private final Optional lateMessageRejectionStartDateTime; - private final Map dynamicAllocationTasksProperties; + @Nullable private final Map dynamicAllocationTasksProperties; public SeekableStreamSupervisorIOConfig( String stream, @@ -62,7 +62,7 @@ public SeekableStreamSupervisorIOConfig( Period completionTimeout, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, - Map dynamicAllocationTasksProperties, + @Nullable Map dynamicAllocationTasksProperties, DateTime lateMessageRejectionStartDateTime ) { @@ -119,6 +119,7 @@ public Integer getReplicas() return replicas; } + @Nullable @JsonProperty public Map getDynamicAllocationTasksProperties() { From b3b75b20992d6a31e6c4012b6408eeaf03192c5b Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 27 Nov 2020 20:10:24 +0800 Subject: [PATCH 11/47] midify test docs --- .../supervisor/SeekableStreamSupervisorStateTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index b089937955b7..5d7e75d5b950 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -840,9 +840,9 @@ private static Map getProperties() dynamicAllocationTasksProperties.put("metricsCollectionIntervalMillis", 500); dynamicAllocationTasksProperties.put("metricsCollectionRangeMillis", 500); dynamicAllocationTasksProperties.put("scaleOutThreshold", 5000000); - dynamicAllocationTasksProperties.put("triggerSaleOutThresholdFrequency", 0.3); + dynamicAllocationTasksProperties.put("triggerScaleOutThresholdFrequency", 0.3); dynamicAllocationTasksProperties.put("scaleInThreshold", 1000000); - dynamicAllocationTasksProperties.put("triggerSaleInThresholdFrequency", 0.8); + dynamicAllocationTasksProperties.put("triggerScaleInThresholdFrequency", 0.8); dynamicAllocationTasksProperties.put("dynamicCheckStartDelayMillis", 0); dynamicAllocationTasksProperties.put("dynamicCheckPeriod", 100); dynamicAllocationTasksProperties.put("taskCountMax", 8); From 18375474208f172c5728a60f3935f4cc023ac981 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 27 Nov 2020 23:37:50 +0800 Subject: [PATCH 12/47] modify docs --- .../extensions-core/kafka-ingestion.md | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 3aed574a53a7..20609a64fc2a 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -150,20 +150,20 @@ A sample supervisor spec is shown below: | Property | Description | Default | | ------------- | ------------- | ------------- | -| enableDynamicAllocationTasks | whether enable this feature or not | false | -| metricsCollectionIntervalMillis | Define the frequency of lag points collection. | 30000 | -| metricsCollectionRangeMillis | The total time window of lag collection, Use with metricsCollectionIntervalMillis，it means that in the recent metricsCollectionRangeMill, collect lag metric points every metricsCollectionIntervalMillis. | 600000 | -| scaleOutThreshold | The Threshold of scale out action | 6000000 | -| triggerScaleOutThresholdFrequency | If 'triggerScaleOutThresholdFrequency' percent of lag points are higher than scaleOutThreshold, then do scale out action. | 0.3 | -| scaleInThreshold | The Threshold of scale in action | 1000000 | -| triggerScaleInThresholdFrequency | If 'triggerScaleInThresholdFrequency' percent of lag points are lower than scaleOutThreshold, then do scale in action. | 0.9 | -| dynamicCheckStartDelayMillis | Number of milliseconds after supervisor starts when first check scale logic. | 300000 | -| dynamicCheckPeriod | the frequency of checking wheather to do scale action | 60000 | -| taskCountMax | Maximum value of task count | 4 | -| taskCountMin | Minimum value of task count | 1 | -| scaleInStep | How many tasks to reduce at once | 1 | -| scaleOutStep | How many tasks to add at a time | 2 | -| minTriggerDynamicFrequencyMillis | Minimum time interval between two scale actions | 600000 | +| `enableDynamicAllocationTasks` | whether enable this feature or not | false | +| `metricsCollectionIntervalMilli`s | Define the frequency of lag points collection. | 30000 | +| `metricsCollectionRangeMillis` | The total time window of lag collection, Use with `metricsCollectionIntervalMillis`，it means that in the recent `metricsCollectionRangeMill`, collect lag metric points every `metricsCollectionIntervalMillis`. | 600000 | +| `scaleOutThreshold` | The Threshold of scale out action | 6000000 | +| `triggerScaleOutThresholdFrequency` | If `triggerScaleOutThresholdFrequency` percent of lag points are higher than `scaleOutThreshold`, then do scale out action. | 0.3 | +| `scaleInThreshold` | The Threshold of scale in action | 1000000 | +| `triggerScaleInThresholdFrequency` | If `triggerScaleInThresholdFrequency` percent of lag points are lower than `scaleOutThreshold`, then do scale in action. | 0.9 | +| `dynamicCheckStartDelayMillis` | Number of milliseconds after supervisor starts when first check scale logic. | 300000 | +| `dynamicCheckPeriod` | the frequency of checking wheather to do scale action | 60000 | +| `taskCountMax` | Maximum value of task count | 4 | +| `taskCountMin` | Minimum value of task count | 1 | +| `scaleInStep` | How many tasks to reduce at once | 1 | +| `scaleOutStep` | How many tasks to add at a time | 2 | +| `minTriggerDynamicFrequencyMillis` | Minimum time interval between two scale actions | 600000 | #### Specifying data format From 50a94cadb70d153e5afc675e19ecf1b08654a8c1 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Sat, 28 Nov 2020 00:25:40 +0800 Subject: [PATCH 13/47] modify docs --- docs/development/extensions-core/kafka-ingestion.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 20609a64fc2a..c1a45a72a688 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -158,7 +158,7 @@ A sample supervisor spec is shown below: | `scaleInThreshold` | The Threshold of scale in action | 1000000 | | `triggerScaleInThresholdFrequency` | If `triggerScaleInThresholdFrequency` percent of lag points are lower than `scaleOutThreshold`, then do scale in action. | 0.9 | | `dynamicCheckStartDelayMillis` | Number of milliseconds after supervisor starts when first check scale logic. | 300000 | -| `dynamicCheckPeriod` | the frequency of checking wheather to do scale action | 60000 | +| `dynamicCheckPeriod` | the frequency of checking weather to do scale action | 60000 | | `taskCountMax` | Maximum value of task count | 4 | | `taskCountMin` | Minimum value of task count | 1 | | `scaleInStep` | How many tasks to reduce at once | 1 | From 4a0d706626ab82ac438cb4c7c3ba89b4b94e653b Mon Sep 17 00:00:00 2001 From: yuezhang Date: Sat, 28 Nov 2020 12:49:59 +0800 Subject: [PATCH 14/47] modify docs --- docs/development/extensions-core/kafka-ingestion.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index c1a45a72a688..52294ce47755 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -144,24 +144,24 @@ A sample supervisor spec is shown below: |`lateMessageRejectionStartDateTime`|ISO8601 DateTime|Configure tasks to reject messages with timestamps earlier than this date time; for example if this is set to `2016-01-01T11:00Z` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline).|no (default == none)| |`lateMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps earlier than this period before the task was created; for example if this is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline). Please note that only one of `lateMessageRejectionPeriod` or `lateMessageRejectionStartDateTime` can be specified.|no (default == none)| |`earlyMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps later than this period after the task reached its taskDuration; for example if this is set to `PT1H`, the taskDuration is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps later than *2016-01-01T14:00Z* will be dropped. **Note:** Tasks sometimes run past their task duration, for example, in cases of supervisor failover. Setting earlyMessageRejectionPeriod too low may cause messages to be dropped unexpectedly whenever a task runs past its originally configured task duration.|no (default == none)| -|`dynamicAllocationTasksProperties`|Object|`dynamicAllocationTasksProperties` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. See below `Dynamic Allocation Tasks Properties` for details.|no (default == null)| +|`dynamicAllocationTasksProperties`|Object|`dynamicAllocationTasksProperties` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. See [Dynamic Allocation Tasks Properties](#Dynamic Allocation Tasks Properties) for details.|no (default == null)| #### Dynamic Allocation Tasks Properties | Property | Description | Default | | ------------- | ------------- | ------------- | | `enableDynamicAllocationTasks` | whether enable this feature or not | false | -| `metricsCollectionIntervalMilli`s | Define the frequency of lag points collection. | 30000 | +| `metricsCollectionIntervalMillis` | Define the frequency of lag points collection. | 30000 | | `metricsCollectionRangeMillis` | The total time window of lag collection, Use with `metricsCollectionIntervalMillis`，it means that in the recent `metricsCollectionRangeMill`, collect lag metric points every `metricsCollectionIntervalMillis`. | 600000 | | `scaleOutThreshold` | The Threshold of scale out action | 6000000 | | `triggerScaleOutThresholdFrequency` | If `triggerScaleOutThresholdFrequency` percent of lag points are higher than `scaleOutThreshold`, then do scale out action. | 0.3 | | `scaleInThreshold` | The Threshold of scale in action | 1000000 | | `triggerScaleInThresholdFrequency` | If `triggerScaleInThresholdFrequency` percent of lag points are lower than `scaleOutThreshold`, then do scale in action. | 0.9 | | `dynamicCheckStartDelayMillis` | Number of milliseconds after supervisor starts when first check scale logic. | 300000 | -| `dynamicCheckPeriod` | the frequency of checking weather to do scale action | 60000 | +| `dynamicCheckPeriod` | the frequency of checking whether to do scale action | 60000 | | `taskCountMax` | Maximum value of task count | 4 | | `taskCountMin` | Minimum value of task count | 1 | -| `scaleInStep` | How many tasks to reduce at once | 1 | +| `scaleInStep` | How many tasks to reduce at a time | 1 | | `scaleOutStep` | How many tasks to add at a time | 2 | | `minTriggerDynamicFrequencyMillis` | Minimum time interval between two scale actions | 600000 | From aa70a5c33c6e1b50739a1b6f15ab3eb67586fb90 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 15 Jan 2021 08:11:13 +0800 Subject: [PATCH 15/47] merge from master --- .../DataSourcesSnapshotBenchmark.java | 124 ++++++ .../druid/annotations/SuppressFBWarnings.java | 42 ++ .../util/metrics/BasicMonitorScheduler.java | 65 ++++ .../ClockDriftSafeMonitorScheduler.java | 134 +++++++ .../druid/metadata/DynamicConfigProvider.java | 39 ++ .../MapStringDynamicConfigProvider.java | 47 +++ .../metrics/BasicMonitorSchedulerTest.java | 111 ++++++ .../MapStringDynamicConfigProviderTest.java | 44 +++ .../docker/DockerfileBuildTarAdvanced | 62 +++ .../extensions-core/druid-aws-rds.md | 38 ++ .../development/extensions-core/kubernetes.md | 88 +++++ docs/operations/dynamic-config-provider.md | 33 ++ .../druid/cluster/data/indexer/jvm.config | 9 + .../druid/cluster/data/indexer/main.config | 1 + .../cluster/data/indexer/runtime.properties | 38 ++ .../druid-aws-rds-extensions/pom.xml | 80 ++++ .../apache/druid/aws/rds/AWSRDSModule.java | 47 +++ .../aws/rds/AWSRDSTokenPasswordProvider.java | 123 ++++++ ...rg.apache.druid.initialization.DruidModule | 16 + .../rds/AWSRDSTokenPasswordProviderTest.java | 82 ++++ .../data/input/kafka/KafkaRecordEntity.java | 53 +++ extensions-core/kubernetes-extensions/pom.xml | 160 ++++++++ .../k8s/discovery/DefaultK8sApiClient.java | 186 +++++++++ .../DefaultK8sLeaderElectorFactory.java | 91 +++++ .../DiscoveryDruidNodeAndResourceVersion.java | 44 +++ .../k8s/discovery/DiscoveryDruidNodeList.java | 52 +++ .../druid/k8s/discovery/K8sApiClient.java | 37 ++ .../k8s/discovery/K8sDiscoveryConfig.java | 205 ++++++++++ .../k8s/discovery/K8sDiscoveryModule.java | 152 ++++++++ .../k8s/discovery/K8sDruidLeaderSelector.java | 152 ++++++++ .../k8s/discovery/K8sDruidNodeAnnouncer.java | 266 +++++++++++++ .../K8sDruidNodeDiscoveryProvider.java | 363 ++++++++++++++++++ .../discovery/K8sLeaderElectorFactory.java | 41 ++ .../discovery/LeaderElectorAsyncWrapper.java | 119 ++++++ .../apache/druid/k8s/discovery/PodInfo.java | 59 +++ .../druid/k8s/discovery/WatchResult.java | 36 ++ ...rg.apache.druid.initialization.DruidModule | 16 + .../K8sAnnouncerAndDiscoveryIntTest.java | 124 ++++++ .../k8s/discovery/K8sDiscoveryConfigTest.java | 79 ++++ .../K8sDruidLeaderElectionIntTest.java | 166 ++++++++ .../discovery/K8sDruidLeaderSelectorTest.java | 180 +++++++++ .../discovery/K8sDruidNodeAnnouncerTest.java | 143 +++++++ .../K8sDruidNodeDiscoveryProviderTest.java | 343 +++++++++++++++++ .../k8s/discovery/NoopServiceEmitter.java | 36 ++ .../indexing/overlord/TaskQueueTest.java | 278 ++++++++++++++ .../docker/docker-compose.cli-indexer.yml | 99 +++++ .../docker-compose.high-availability.yml | 151 ++++++++ .../docker/environment-configs/indexer | 38 ++ .../high-availability-sample-data.sql | 20 + integration-tests/k8s/role-and-binding.yaml | 39 ++ integration-tests/k8s/tiny-cluster.yaml | 347 +++++++++++++++++ integration-tests/k8s_run_config_file.json | 16 + .../script/docker_compose_args.sh | 64 +++ .../script/setup_druid_on_k8s.sh | 52 +++ .../script/setup_druid_operator_on_k8s.sh | 37 ++ integration-tests/script/setup_k8s_cluster.sh | 34 ++ integration-tests/script/stop_k8s_cluster.sh | 19 + .../apache/druid/cli/CliCustomNodeRole.java | 166 ++++++++ .../cli/CustomNodeRoleCommandCreator.java | 31 ++ .../tests/indexer/ITHttpInputSourceTest.java | 53 +++ .../leadership/ITHighAvailabilityTest.java | 300 +++++++++++++++ .../wikipedia_http_inputsource_queries.json | 98 +++++ .../wikipedia_http_inputsource_task.json | 90 +++++ ...ia_index_with_merge_column_limit_task.json | 86 +++++ ...cast_join_after_drop_metadata_queries.json | 9 + .../queries/high_availability_sys.json | 39 ++ licenses/bin/json-bigint-native.MIT | 20 + .../druid/query/BadJsonQueryException.java | 45 +++ .../apache/druid/query/BadQueryException.java | 40 ++ .../GroupingAggregatorFactory.java | 316 +++++++++++++++ .../constant/LongConstantAggregator.java | 66 ++++ .../LongConstantBufferAggregator.java | 74 ++++ .../LongConstantVectorAggregator.java | 69 ++++ .../segment/DimensionHandlerProvider.java | 26 ++ .../vector/ReadableVectorInspector.java | 38 ++ .../GroupingAggregatorFactoryTest.java | 175 +++++++++ .../constant/LongConstantAggregatorTest.java | 63 +++ .../LongConstantBufferAggregatorTest.java | 70 ++++ .../LongConstantVectorAggregatorTest.java | 65 ++++ .../segment/DimensionHandlerUtilsTest.java | 46 +++ .../segment/column/ColumnBuilderTest.java | 38 ++ .../incremental/IncrementalIndexCreator.java | 244 ++++++++++++ .../OffheapIncrementalIndexTestSpec.java | 107 ++++++ .../ExpressionVectorObjectSelectorTest.java | 70 ++++ .../ExpressionVectorValueSelectorTest.java | 116 ++++++ .../druid/discovery/BaseNodeRoleWatcher.java | 301 +++++++++++++++ .../druid/metadata/BasicDataSourceExt.java | 179 +++++++++ .../HttpIndexingServiceClientTest.java | 164 ++++++++ .../discovery/BaseNodeRoleWatcherTest.java | 161 ++++++++ .../metadata/BasicDataSourceExtTest.java | 113 ++++++ ...nifiedIndexerAppenderatorsManagerTest.java | 116 ++++++ .../druid/sql/SqlPlanningException.java | 82 ++++ .../builtin/GroupingSqlAggregator.java | 126 ++++++ web-console/src/bootstrap/ace.ts | 24 ++ .../numeric-input-with-default.spec.tsx.snap | 19 + .../numeric-input-with-default.spec.tsx | 30 ++ .../numeric-input-with-default.tsx | 51 +++ web-console/src/singletons/api.spec.ts | 54 +++ web-console/src/singletons/api.ts | 59 +++ web-console/src/singletons/index.ts | 21 + 100 files changed, 9580 insertions(+) create mode 100644 benchmarks/src/test/java/org/apache/druid/benchmark/DataSourcesSnapshotBenchmark.java create mode 100644 core/src/main/java/org/apache/druid/annotations/SuppressFBWarnings.java create mode 100644 core/src/main/java/org/apache/druid/java/util/metrics/BasicMonitorScheduler.java create mode 100644 core/src/main/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorScheduler.java create mode 100644 core/src/main/java/org/apache/druid/metadata/DynamicConfigProvider.java create mode 100644 core/src/main/java/org/apache/druid/metadata/MapStringDynamicConfigProvider.java create mode 100644 core/src/test/java/org/apache/druid/java/util/metrics/BasicMonitorSchedulerTest.java create mode 100644 core/src/test/java/org/apache/druid/metadata/MapStringDynamicConfigProviderTest.java create mode 100644 distribution/docker/DockerfileBuildTarAdvanced create mode 100644 docs/development/extensions-core/druid-aws-rds.md create mode 100644 docs/development/extensions-core/kubernetes.md create mode 100644 docs/operations/dynamic-config-provider.md create mode 100644 examples/conf/druid/cluster/data/indexer/jvm.config create mode 100644 examples/conf/druid/cluster/data/indexer/main.config create mode 100644 examples/conf/druid/cluster/data/indexer/runtime.properties create mode 100644 extensions-core/druid-aws-rds-extensions/pom.xml create mode 100644 extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSModule.java create mode 100644 extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java create mode 100644 extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule create mode 100644 extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java create mode 100644 extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafka/KafkaRecordEntity.java create mode 100644 extensions-core/kubernetes-extensions/pom.xml create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DefaultK8sApiClient.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DefaultK8sLeaderElectorFactory.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DiscoveryDruidNodeAndResourceVersion.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DiscoveryDruidNodeList.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sApiClient.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDiscoveryConfig.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDiscoveryModule.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidLeaderSelector.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeDiscoveryProvider.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sLeaderElectorFactory.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/LeaderElectorAsyncWrapper.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/PodInfo.java create mode 100644 extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/WatchResult.java create mode 100644 extensions-core/kubernetes-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule create mode 100644 extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sAnnouncerAndDiscoveryIntTest.java create mode 100644 extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDiscoveryConfigTest.java create mode 100644 extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidLeaderElectionIntTest.java create mode 100644 extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidLeaderSelectorTest.java create mode 100644 extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncerTest.java create mode 100644 extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidNodeDiscoveryProviderTest.java create mode 100644 extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/NoopServiceEmitter.java create mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java create mode 100644 integration-tests/docker/docker-compose.cli-indexer.yml create mode 100644 integration-tests/docker/docker-compose.high-availability.yml create mode 100644 integration-tests/docker/environment-configs/indexer create mode 100644 integration-tests/docker/test-data/high-availability-sample-data.sql create mode 100644 integration-tests/k8s/role-and-binding.yaml create mode 100644 integration-tests/k8s/tiny-cluster.yaml create mode 100644 integration-tests/k8s_run_config_file.json create mode 100644 integration-tests/script/docker_compose_args.sh create mode 100755 integration-tests/script/setup_druid_on_k8s.sh create mode 100755 integration-tests/script/setup_druid_operator_on_k8s.sh create mode 100755 integration-tests/script/setup_k8s_cluster.sh create mode 100755 integration-tests/script/stop_k8s_cluster.sh create mode 100644 integration-tests/src/main/java/org/apache/druid/cli/CliCustomNodeRole.java create mode 100644 integration-tests/src/main/java/org/apache/druid/cli/CustomNodeRoleCommandCreator.java create mode 100644 integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHttpInputSourceTest.java create mode 100644 integration-tests/src/test/java/org/apache/druid/tests/leadership/ITHighAvailabilityTest.java create mode 100644 integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_queries.json create mode 100644 integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_task.json create mode 100644 integration-tests/src/test/resources/indexer/wikipedia_index_with_merge_column_limit_task.json create mode 100644 integration-tests/src/test/resources/queries/broadcast_join_after_drop_metadata_queries.json create mode 100644 integration-tests/src/test/resources/queries/high_availability_sys.json create mode 100644 licenses/bin/json-bigint-native.MIT create mode 100644 processing/src/main/java/org/apache/druid/query/BadJsonQueryException.java create mode 100644 processing/src/main/java/org/apache/druid/query/BadQueryException.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/GroupingAggregatorFactory.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantAggregator.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantBufferAggregator.java create mode 100644 processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantVectorAggregator.java create mode 100644 processing/src/main/java/org/apache/druid/segment/DimensionHandlerProvider.java create mode 100644 processing/src/main/java/org/apache/druid/segment/vector/ReadableVectorInspector.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/GroupingAggregatorFactoryTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantAggregatorTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantBufferAggregatorTest.java create mode 100644 processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantVectorAggregatorTest.java create mode 100644 processing/src/test/java/org/apache/druid/segment/DimensionHandlerUtilsTest.java create mode 100644 processing/src/test/java/org/apache/druid/segment/column/ColumnBuilderTest.java create mode 100644 processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexCreator.java create mode 100644 processing/src/test/java/org/apache/druid/segment/incremental/OffheapIncrementalIndexTestSpec.java create mode 100644 processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java create mode 100644 processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelectorTest.java create mode 100644 server/src/main/java/org/apache/druid/discovery/BaseNodeRoleWatcher.java create mode 100644 server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java create mode 100644 server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java create mode 100644 server/src/test/java/org/apache/druid/discovery/BaseNodeRoleWatcherTest.java create mode 100644 server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java create mode 100644 server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java create mode 100644 sql/src/main/java/org/apache/druid/sql/SqlPlanningException.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/GroupingSqlAggregator.java create mode 100644 web-console/src/bootstrap/ace.ts create mode 100644 web-console/src/components/numeric-input-with-default/__snapshots__/numeric-input-with-default.spec.tsx.snap create mode 100644 web-console/src/components/numeric-input-with-default/numeric-input-with-default.spec.tsx create mode 100644 web-console/src/components/numeric-input-with-default/numeric-input-with-default.tsx create mode 100644 web-console/src/singletons/api.spec.ts create mode 100644 web-console/src/singletons/api.ts create mode 100644 web-console/src/singletons/index.ts diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DataSourcesSnapshotBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DataSourcesSnapshotBenchmark.java new file mode 100644 index 000000000000..ab6fff0186e0 --- /dev/null +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DataSourcesSnapshotBenchmark.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.benchmark; + +import org.apache.druid.client.DataSourcesSnapshot; +import org.apache.druid.client.ImmutableDruidDataSource; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.timeline.DataSegment; +import org.apache.druid.timeline.partition.NoneShardSpec; +import org.joda.time.Interval; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +@State(Scope.Benchmark) +@Fork(1) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 10) +@Measurement(iterations = 50) +public class DataSourcesSnapshotBenchmark +{ + private static Interval TEST_SEGMENT_INTERVAL = Intervals.of("2012-03-15T00:00:00.000/2012-03-16T00:00:00.000"); + + @Param({"500", "1000"}) + private int numDataSources; + + @Param({"1000", "2000"}) + private int numSegmentPerDataSource; + + private DataSourcesSnapshot snapshot; + + @Setup + public void setUp() + { + long start = System.currentTimeMillis(); + + Map dataSources = new HashMap<>(); + + for (int i = 0; i < numDataSources; i++) { + String dataSource = StringUtils.format("ds-%d", i); + List segments = new ArrayList<>(); + + for (int j = 0; j < numSegmentPerDataSource; j++) { + segments.add( + new DataSegment( + dataSource, + TEST_SEGMENT_INTERVAL, + String.valueOf(j), + Collections.emptyMap(), + Collections.emptyList(), + Collections.emptyList(), + NoneShardSpec.instance(), + 0, + 10L + ) + ); + } + + dataSources.put(dataSource, new ImmutableDruidDataSource(dataSource, Collections.emptyMap(), segments)); + } + + snapshot = new DataSourcesSnapshot(dataSources); + + System.out.println("Setup Time " + (System.currentTimeMillis() - start) + " ms"); + } + + @Benchmark + public void iterateUsing_iterateAllUsedSegmentsInSnapshot(Blackhole blackhole) + { + long totalSize = 0; + for (DataSegment segment : snapshot.iterateAllUsedSegmentsInSnapshot()) { + totalSize += segment.getSize(); + } + blackhole.consume(totalSize); + } + + @Benchmark + public void iterateUsing_forloops(Blackhole blackhole) + { + long totalSize = 0; + for (ImmutableDruidDataSource dataSource : snapshot.getDataSourcesWithAllUsedSegments()) { + for (DataSegment segment : dataSource.getSegments()) { + totalSize += segment.getSize(); + } + } + blackhole.consume(totalSize); + } +} + diff --git a/core/src/main/java/org/apache/druid/annotations/SuppressFBWarnings.java b/core/src/main/java/org/apache/druid/annotations/SuppressFBWarnings.java new file mode 100644 index 000000000000..fb1d5b1a126d --- /dev/null +++ b/core/src/main/java/org/apache/druid/annotations/SuppressFBWarnings.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.annotations; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Annotation for suppressing spotbugs checks when necessary. + */ +@Retention(RetentionPolicy.CLASS) +public @interface SuppressFBWarnings +{ + /** + * The set of FindBugs warnings that are to be suppressed in + * annotated element. The value can be a bug category, kind or pattern. + * + */ + String[] value() default {}; + + /** + * Optional documentation of the reason why the warning is suppressed + */ + String justification() default ""; +} diff --git a/core/src/main/java/org/apache/druid/java/util/metrics/BasicMonitorScheduler.java b/core/src/main/java/org/apache/druid/java/util/metrics/BasicMonitorScheduler.java new file mode 100644 index 000000000000..e3e9572e573c --- /dev/null +++ b/core/src/main/java/org/apache/druid/java/util/metrics/BasicMonitorScheduler.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.java.util.metrics; + +import org.apache.druid.java.util.common.concurrent.ScheduledExecutors; +import org.apache.druid.java.util.common.concurrent.ScheduledExecutors.Signal; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; + +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; + +/** + * A {@link MonitorScheduler} implementation based on {@link ScheduledExecutorService}. + */ +public class BasicMonitorScheduler extends MonitorScheduler +{ + private final ScheduledExecutorService exec; + + public BasicMonitorScheduler( + MonitorSchedulerConfig config, + ServiceEmitter emitter, + List monitors, + ScheduledExecutorService exec + ) + { + super(config, emitter, monitors); + this.exec = exec; + } + + @Override + void startMonitor(Monitor monitor) + { + monitor.start(); + ScheduledExecutors.scheduleAtFixedRate( + exec, + getConfig().getEmitterPeriod(), + () -> { + // Run one more time even if the monitor was removed, in case there's some extra data to flush + if (monitor.monitor(getEmitter()) && hasMonitor(monitor)) { + return Signal.REPEAT; + } else { + removeMonitor(monitor); + return Signal.STOP; + } + } + ); + } +} diff --git a/core/src/main/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorScheduler.java b/core/src/main/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorScheduler.java new file mode 100644 index 000000000000..6fcb24406971 --- /dev/null +++ b/core/src/main/java/org/apache/druid/java/util/metrics/ClockDriftSafeMonitorScheduler.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.java.util.metrics; + +import io.timeandspace.cronscheduler.CronScheduler; +import io.timeandspace.cronscheduler.CronTask; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** + * A {@link MonitorScheduler} implementation based on {@link CronScheduler}. + */ +public class ClockDriftSafeMonitorScheduler extends MonitorScheduler +{ + private static final Logger LOG = new Logger(ClockDriftSafeMonitorScheduler.class); + + private final CronScheduler monitorScheduler; + private final ExecutorService monitorRunner; + + public ClockDriftSafeMonitorScheduler( + MonitorSchedulerConfig config, + ServiceEmitter emitter, + List monitors, + CronScheduler monitorScheduler, + ExecutorService monitorRunner + ) + { + super(config, emitter, monitors); + this.monitorScheduler = monitorScheduler; + this.monitorRunner = monitorRunner; + } + + @Override + void startMonitor(final Monitor monitor) + { + monitor.start(); + long rate = getConfig().getEmitterPeriod().getMillis(); + final AtomicReference> futureReference = new AtomicReference<>(); + Future future = monitorScheduler.scheduleAtFixedRate( + rate, + rate, + TimeUnit.MILLISECONDS, + new CronTask() + { + private Future cancellationFuture = null; + private Future monitorFuture = null; + + @Override + public void run(long scheduledRunTimeMillis) + { + waitForScheduleFutureToBeSet(); + if (cancellationFuture == null) { + LOG.error("scheduleFuture is not set. Can't run monitor[%s]", monitor.getClass().getName()); + return; + } + try { + // Do nothing if the monitor is still running. + if (monitorFuture == null || monitorFuture.isDone()) { + if (monitorFuture != null) { + // monitorFuture must be done at this moment if it's not null + if (!(monitorFuture.get() && hasMonitor(monitor))) { + stopMonitor(monitor); + return; + } + } + + LOG.trace("Running monitor[%s]", monitor.getClass().getName()); + monitorFuture = monitorRunner.submit(() -> { + try { + return monitor.monitor(getEmitter()); + } + catch (Throwable e) { + LOG.error( + e, + "Exception while executing monitor[%s]. Rescheduling in %s ms", + monitor.getClass().getName(), + rate + ); + return Boolean.TRUE; + } + }); + } + } + catch (Throwable e) { + LOG.error(e, "Uncaught exception."); + } + } + + private void waitForScheduleFutureToBeSet() + { + if (cancellationFuture == null) { + while (!Thread.currentThread().isInterrupted()) { + if (futureReference.get() != null) { + cancellationFuture = futureReference.get(); + break; + } + } + } + } + + private void stopMonitor(Monitor monitor) + { + removeMonitor(monitor); + cancellationFuture.cancel(false); + LOG.debug("Stopped monitor[%s]", monitor.getClass().getName()); + } + } + ); + futureReference.set(future); + } +} diff --git a/core/src/main/java/org/apache/druid/metadata/DynamicConfigProvider.java b/core/src/main/java/org/apache/druid/metadata/DynamicConfigProvider.java new file mode 100644 index 000000000000..52f032a0a195 --- /dev/null +++ b/core/src/main/java/org/apache/druid/metadata/DynamicConfigProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.metadata; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.guice.annotations.ExtensionPoint; + +import java.util.Map; + +/** + * This is used to get [secure] configuration in various places in an extensible way. + */ +@ExtensionPoint +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = MapStringDynamicConfigProvider.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "mapString", value = MapStringDynamicConfigProvider.class), +}) +public interface DynamicConfigProvider +{ + Map getConfig(); +} diff --git a/core/src/main/java/org/apache/druid/metadata/MapStringDynamicConfigProvider.java b/core/src/main/java/org/apache/druid/metadata/MapStringDynamicConfigProvider.java new file mode 100644 index 000000000000..1ef5a15e8a97 --- /dev/null +++ b/core/src/main/java/org/apache/druid/metadata/MapStringDynamicConfigProvider.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.metadata; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +public class MapStringDynamicConfigProvider implements DynamicConfigProvider +{ + private final ImmutableMap config; + + @JsonCreator + public MapStringDynamicConfigProvider( + @JsonProperty("config") Map config + ) + { + this.config = ImmutableMap.copyOf(config); + } + + + @Override + @JsonProperty + public Map getConfig() + { + return config; + } +} diff --git a/core/src/test/java/org/apache/druid/java/util/metrics/BasicMonitorSchedulerTest.java b/core/src/test/java/org/apache/druid/java/util/metrics/BasicMonitorSchedulerTest.java new file mode 100644 index 000000000000..ea0408050ef9 --- /dev/null +++ b/core/src/test/java/org/apache/druid/java/util/metrics/BasicMonitorSchedulerTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.java.util.metrics; + +import com.google.common.collect.ImmutableList; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.joda.time.Duration; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +import java.util.concurrent.ScheduledExecutorService; + +public class BasicMonitorSchedulerTest +{ + private final MonitorSchedulerConfig config = new MonitorSchedulerConfig() + { + @Override + public Duration getEmitterPeriod() + { + return Duration.millis(5); + } + }; + private ServiceEmitter emitter; + private ScheduledExecutorService exec; + + @Before + public void setup() + { + emitter = Mockito.mock(ServiceEmitter.class); + exec = Execs.scheduledSingleThreaded("BasicMonitorSchedulerTest"); + } + + @Test + public void testStart_RepeatScheduling() throws InterruptedException + { + final Monitor monitor = Mockito.mock(Monitor.class); + Mockito.when(monitor.monitor(ArgumentMatchers.any())).thenReturn(true); + + final BasicMonitorScheduler scheduler = new BasicMonitorScheduler( + config, + emitter, + ImmutableList.of(monitor), + exec + ); + scheduler.start(); + Thread.sleep(100); + Mockito.verify(monitor, Mockito.atLeast(2)).monitor(ArgumentMatchers.any()); + scheduler.stop(); + } + + @Test + public void testStart_RepeatAndStopScheduling() throws InterruptedException + { + final Monitor monitor = Mockito.mock(Monitor.class); + Mockito.when(monitor.monitor(ArgumentMatchers.any())).thenReturn(true, true, true, false); + + final BasicMonitorScheduler scheduler = new BasicMonitorScheduler( + config, + emitter, + ImmutableList.of(monitor), + exec + ); + scheduler.start(); + Thread.sleep(100); + // monitor.monitor() is called 5 times since a new task is scheduled first and then the current one is executed. + // See ScheduledExecutors.scheduleAtFixedRate() for details. + Mockito.verify(monitor, Mockito.times(5)).monitor(ArgumentMatchers.any()); + scheduler.stop(); + } + + @Test + public void testStart_UnexpectedExceptionWhileMonitoring_ContinueMonitor() throws InterruptedException + { + final Monitor monitor = Mockito.mock(Monitor.class); + Mockito.when(monitor.monitor(ArgumentMatchers.any())) + .thenThrow(new RuntimeException("Test throwing exception while monitoring")); + + final BasicMonitorScheduler scheduler = new BasicMonitorScheduler( + config, + emitter, + ImmutableList.of(monitor), + exec + ); + scheduler.start(); + Thread.sleep(100); + // monitor.monitor() is called 5 times since a new task is scheduled first and then the current one is executed. + // See ScheduledExecutors.scheduleAtFixedRate() for details. + Mockito.verify(monitor, Mockito.atLeast(2)).monitor(ArgumentMatchers.any()); + scheduler.stop(); + } +} diff --git a/core/src/test/java/org/apache/druid/metadata/MapStringDynamicConfigProviderTest.java b/core/src/test/java/org/apache/druid/metadata/MapStringDynamicConfigProviderTest.java new file mode 100644 index 000000000000..cdf46e12b36f --- /dev/null +++ b/core/src/test/java/org/apache/druid/metadata/MapStringDynamicConfigProviderTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.metadata; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import org.junit.Assert; +import org.junit.Test; + +public class MapStringDynamicConfigProviderTest +{ + @Test + public void testSerde() throws Exception + { + DynamicConfigProvider original = new MapStringDynamicConfigProvider(ImmutableMap.of("k", "v")); + + ObjectMapper jsonMapper = new ObjectMapper(); + + MapStringDynamicConfigProvider recreated = (MapStringDynamicConfigProvider) jsonMapper.readValue( + jsonMapper.writeValueAsString(original), + DynamicConfigProvider.class + ); + + Assert.assertEquals(1, recreated.getConfig().size()); + Assert.assertEquals("v", recreated.getConfig().get("k")); + } +} diff --git a/distribution/docker/DockerfileBuildTarAdvanced b/distribution/docker/DockerfileBuildTarAdvanced new file mode 100644 index 000000000000..a58087f4e51a --- /dev/null +++ b/distribution/docker/DockerfileBuildTarAdvanced @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Using For IT on K8s +FROM maven:3-jdk-8-slim as builder + +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get -qq update \ + && apt-get -qq -y install --no-install-recommends python3 python3-yaml + +COPY . /src +WORKDIR /src + +RUN VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.1.1:evaluate \ + -Dexpression=project.version -DforceStdout=true \ + ) \ + && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \ + && ln -s /opt/apache-druid-${VERSION} /opt/druid + +FROM amd64/busybox:1.30.0-glibc as busybox + +FROM gcr.io/distroless/java:8 +LABEL maintainer="Apache Druid Developers " + +COPY --from=busybox /bin/busybox /busybox/busybox +RUN ["/busybox/busybox", "--install", "/bin"] + +RUN mkdir -p /shared/wikiticker-it + +COPY examples/quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz /shared/wikiticker-it/wikiticker-2015-09-12-sampled.json.gz +COPY integration-tests/docker/wiki-simple-lookup.json /shared/wikiticker-it/wiki-simple-lookup.json + +RUN addgroup -S -g 1000 druid \ + && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid \ + && mkdir -p /opt/druid/var \ + && chown -R druid:druid /opt \ + && chmod 775 /opt/druid/var + +COPY --chown=druid:druid --from=builder /opt /opt +COPY distribution/docker/druid.sh /druid.sh + +USER druid +VOLUME /opt/druid/var +WORKDIR /opt/druid + +ENTRYPOINT ["/druid.sh"] diff --git a/docs/development/extensions-core/druid-aws-rds.md b/docs/development/extensions-core/druid-aws-rds.md new file mode 100644 index 000000000000..6bb0cd826be4 --- /dev/null +++ b/docs/development/extensions-core/druid-aws-rds.md @@ -0,0 +1,38 @@ +--- +id: druid-aws-rds +title: "Druid AWS RDS Module" +--- + + + +[AWS RDS](https://aws.amazon.com/rds/) is a managed service to operate relation databases such as PostgreSQL, Mysql etc. These databases could be accessed using static db password mechanism or via [AWS IAM](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html) temporary tokens. This module provides AWS RDS token [password provider](../../operations/password-provider.md) implementation to be used with [mysql-metadata-store](mysql.md) or [postgresql-metadata-store](postgresql.md) when mysql/postgresql is operated using AWS RDS. + +```json +{ "type": "aws-rds-token", "user": "USER", "host": "HOST", "port": PORT, "region": "AWS_REGION" } +``` + +Before using this password provider, please make sure that you have connected all dots for db user to connect using token. +See [AWS Guide](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html). + +To use this extension, make sure you [include](../../development/extensions.md#loading-extensions) it in your config file along with other extensions e.g. + +``` +druid.extensions.loadList=["druid-aws-rds-extensions", "postgresql-metadata-storage", ...] +``` diff --git a/docs/development/extensions-core/kubernetes.md b/docs/development/extensions-core/kubernetes.md new file mode 100644 index 000000000000..daab608890c2 --- /dev/null +++ b/docs/development/extensions-core/kubernetes.md @@ -0,0 +1,88 @@ +--- +id: druid-kubernetes +title: "Kubernetes" +--- + + + +Consider this an [EXPERIMENTAL](../experimental.md) feature mostly because it has not been tested yet on a wide variety of long running Druid clusters. + +Apache Druid Extension to enable using Kubernetes API Server for node discovery and leader election. This extension allows Druid cluster deployment on Kubernetes without Zookeeper. It allows running multiple Druid clusters within same Kubernetes Cluster, See `clusterIdentifier` config below. + + +## Configuration + +To use this extension please make sure to [include](../../development/extensions.md#loading-extensions) `druid-kubernetes-extensions` as an extension. + +This extension works together with HTTP based segment and task management in Druid. Consequently, following configurations must be set on all Druid nodes. + +`druid.zk.service.enabled=false` +`druid.serverview.type=http` +`druid.coordinator.loadqueuepeon.type=http` +`druid.indexer.runner.type=httpRemote` +`druid.discovery.type=k8s` + +For Node Discovery, Each Druid process running inside a pod "announces" itself by adding few "labels" and "annotations" in the pod spec. Druid process needs to be aware of pod name and namespace which it reads from environment variables `POD_NAME` and `POD_NAMESPACE`. These variable names can be changed, see configuration below. But in the end, each pod needs to have self pod name and namespace added as environment variables. + +Additionally, this extension has following configuration. + +### Properties +|Property|Possible Values|Description|Default|required| +|--------|---------------|-----------|-------|--------| +|`druid.discovery.k8s.clusterIdentifier`|`string that matches [a-z0-9][a-z0-9-]*[a-z0-9]`|Unique identifier for this Druid cluster in Kubernetes e.g. us-west-prod-druid.|None|Yes| +|`druid.discovery.k8s.podNameEnvKey`|`Pod Env Variable`|Pod Env variable whose value is that pod's name.|POD_NAME|No| +|`druid.discovery.k8s.podNamespaceEnvKey`|`Pod Env Variable`|Pod Env variable whose value is that pod's kubernetes namespace.|POD_NAMESPACE|No| +|`druid.discovery.k8s.coordinatorLeaderElectionConfigMapNamespace`|`k8s namespace`|Leader election algorithm requires creating a ConfigMap resource in a namespace. This MUST only be provided if different coordinator pods run in different namespaces, such setup is discouraged however.|coordinator pod's namespace|No| +|`druid.discovery.k8s.overlordLeaderElectionConfigMapNamespace`|`k8s namespace`|Leader election algorithm requires creating a ConfigMap resource in a namespace. This MUST only be provided if different overlord pods run in different namespaces, such setup is discouraged however.|overlord pod's namespace|No| +|`druid.discovery.k8s.leaseDuration`|`Duration`|Lease duration used by Leader Election algorithm. Candidates wait for this time before taking over previous Leader.|PT60S|No| +|`druid.discovery.k8s.renewDeadline`|`Duration`|Lease renewal period used by Leader.|PT17S|No| +|`druid.discovery.k8s.retryPeriod`|`Duration`|Retry wait used by Leader Election algorithm on failed operations.|PT5S|No| + +### Gotchas + +- Label/Annotation path in each pod spec MUST EXIST, which is easily satisfied if there is at least one label/annotation in the pod spec already. This limitation may be removed in future. +- Druid Pods need permissions to be able to add labels to self-pod, List and Watch other Pods, create ConfigMap for leader election. Assuming, "default" service account is used by Druid pods, you might need to add following or something similar Kubernetes Role and Role Binding. + +``` +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: druid-cluster +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - '*' +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: druid-cluster +subjects: +- kind: ServiceAccount + name: default +roleRef: + kind: Role + name: druid-cluster + apiGroup: rbac.authorization.k8s.io +``` \ No newline at end of file diff --git a/docs/operations/dynamic-config-provider.md b/docs/operations/dynamic-config-provider.md new file mode 100644 index 000000000000..33bf96639322 --- /dev/null +++ b/docs/operations/dynamic-config-provider.md @@ -0,0 +1,33 @@ +--- +id: dynamic-config-provider +title: "Dynamic Config Providers" +--- + + + +Druid's core mechanism of supplying multiple related set of credentials/secrets/configurations via Druid extension mechanism. Currently, it is only supported for providing Kafka Consumer configuration in [Kafka Ingestion](../development/extensions-core/kafka-ingestion.md). + +Eventually this will replace [PasswordProvider](./password-provider.md) + + +Users can create custom extension of the `DynamicConfigProvider` interface that is registered at Druid process startup. + +For more information, see [Adding a new DynamicConfigProvider implementation](../development/modules.md#adding-a-new-dynamicconfigprovider-implementation). + diff --git a/examples/conf/druid/cluster/data/indexer/jvm.config b/examples/conf/druid/cluster/data/indexer/jvm.config new file mode 100644 index 000000000000..4611a65196a2 --- /dev/null +++ b/examples/conf/druid/cluster/data/indexer/jvm.config @@ -0,0 +1,9 @@ +-server +-Xms4g +-Xmx4g +-XX:MaxDirectMemorySize=4g +-XX:+ExitOnOutOfMemoryError +-Duser.timezone=UTC +-Dfile.encoding=UTF-8 +-Djava.io.tmpdir=var/tmp +-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager diff --git a/examples/conf/druid/cluster/data/indexer/main.config b/examples/conf/druid/cluster/data/indexer/main.config new file mode 100644 index 000000000000..5183399415cf --- /dev/null +++ b/examples/conf/druid/cluster/data/indexer/main.config @@ -0,0 +1 @@ +org.apache.druid.cli.Main server indexer diff --git a/examples/conf/druid/cluster/data/indexer/runtime.properties b/examples/conf/druid/cluster/data/indexer/runtime.properties new file mode 100644 index 000000000000..b36c9eb42872 --- /dev/null +++ b/examples/conf/druid/cluster/data/indexer/runtime.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +druid.service=druid/indexer +druid.plaintextPort=8091 + +# Number of tasks per indexer +druid.worker.capacity=4 + +# Task launch parameters +druid.indexer.task.baseTaskDir=var/druid/task + +# HTTP server threads +druid.server.http.numThreads=60 + +# Processing threads and buffers on Indexer +druid.processing.numMergeBuffers=2 +druid.processing.buffer.sizeBytes=100MiB +druid.processing.numThreads=4 + +# Hadoop indexing +druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp diff --git a/extensions-core/druid-aws-rds-extensions/pom.xml b/extensions-core/druid-aws-rds-extensions/pom.xml new file mode 100644 index 000000000000..397039e36eaa --- /dev/null +++ b/extensions-core/druid-aws-rds-extensions/pom.xml @@ -0,0 +1,80 @@ + + + + + 4.0.0 + + org.apache.druid.extensions + druid-aws-rds-extensions + druid-aws-rds-extensions + druid-aws-rds-extensions + + + org.apache.druid + druid + 0.21.0-SNAPSHOT + ../../pom.xml + + + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + com.amazonaws + aws-java-sdk-rds + ${aws.sdk.version} + + + com.google.guava + guava + provided + + + com.google.inject + guice + provided + + + com.amazonaws + aws-java-sdk-core + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + junit + junit + test + + + diff --git a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSModule.java b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSModule.java new file mode 100644 index 000000000000..298792a09b6b --- /dev/null +++ b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSModule.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.aws.rds; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import org.apache.druid.initialization.DruidModule; + +import java.util.List; + +public class AWSRDSModule implements DruidModule +{ + @Override + public List getJacksonModules() + { + return ImmutableList.of( + new SimpleModule("DruidAwsRdsExtentionsModule").registerSubtypes( + new NamedType(AWSRDSTokenPasswordProvider.class, "aws-rds-token") + ) + ); + } + + @Override + public void configure(Binder binder) + { + } +} diff --git a/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java new file mode 100644 index 000000000000..9c54d45124cb --- /dev/null +++ b/extensions-core/druid-aws-rds-extensions/src/main/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProvider.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.aws.rds; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.rds.auth.GetIamAuthTokenRequest; +import com.amazonaws.services.rds.auth.RdsIamAuthTokenGenerator; +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.metadata.PasswordProvider; + +/** + * Generates the AWS token same as aws cli + * aws rds generate-db-auth-token --hostname HOST --port PORT --region REGION --username USER + * and returns that as password. + * + * Before using this, please make sure that you have connected all dots for db user to connect using token. + * See https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/UsingWithRDS.IAMDBAuth.html + */ +public class AWSRDSTokenPasswordProvider implements PasswordProvider +{ + private static final Logger LOGGER = new Logger(AWSRDSTokenPasswordProvider.class); + private final String user; + private final String host; + private final int port; + private final String region; + + private final AWSCredentialsProvider awsCredentialsProvider; + + @JsonCreator + public AWSRDSTokenPasswordProvider( + @JsonProperty("user") String user, + @JsonProperty("host") String host, + @JsonProperty("port") int port, + @JsonProperty("region") String region, + @JacksonInject AWSCredentialsProvider awsCredentialsProvider + ) + { + this.user = Preconditions.checkNotNull(user, "null metadataStorage user"); + this.host = Preconditions.checkNotNull(host, "null metadataStorage host"); + Preconditions.checkArgument(port > 0, "must provide port"); + this.port = port; + + this.region = Preconditions.checkNotNull(region, "null region"); + + LOGGER.info("AWS RDS Config user[%s], host[%s], port[%d], region[%s]", this.user, this.host, port, this.region); + this.awsCredentialsProvider = Preconditions.checkNotNull(awsCredentialsProvider, "null AWSCredentialsProvider"); + } + + @JsonProperty + public String getUser() + { + return user; + } + + @JsonProperty + public String getHost() + { + return host; + } + + @JsonProperty + public int getPort() + { + return port; + } + + @JsonProperty + public String getRegion() + { + return region; + } + + @JsonIgnore + @Override + public String getPassword() + { + try { + RdsIamAuthTokenGenerator generator = RdsIamAuthTokenGenerator + .builder() + .credentials(awsCredentialsProvider) + .region(region) + .build(); + + String authToken = generator.getAuthToken( + GetIamAuthTokenRequest + .builder() + .hostname(host) + .port(port) + .userName(user) + .build() + ); + + return authToken; + } + catch (Exception ex) { + LOGGER.error(ex, "Couldn't generate AWS token."); + throw new RE(ex, "Couldn't generate AWS token."); + } + } +} diff --git a/extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..93b5bbc38faa --- /dev/null +++ b/extensions-core/druid-aws-rds-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.aws.rds.AWSRDSModule diff --git a/extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java b/extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java new file mode 100644 index 000000000000..c49a7862cbcb --- /dev/null +++ b/extensions-core/druid-aws-rds-extensions/src/test/java/org/apache/druid/aws/rds/AWSRDSTokenPasswordProviderTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.aws.rds; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.fasterxml.jackson.databind.InjectableValues; +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.metadata.PasswordProvider; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +public class AWSRDSTokenPasswordProviderTest +{ + @Test + public void testSerde() throws IOException + { + ObjectMapper jsonMapper = new ObjectMapper(); + + for (Module module : new AWSRDSModule().getJacksonModules()) { + jsonMapper.registerModule(module); + } + + jsonMapper.setInjectableValues( + new InjectableValues.Std().addValue(AWSCredentialsProvider.class, new AWSCredentialsProvider() + { + @Override + public AWSCredentials getCredentials() + { + return null; + } + + @Override + public void refresh() + { + + } + }) + ); + + String jsonStr = "{\n" + + " \"type\": \"aws-rds-token\",\n" + + " \"user\": \"testuser\",\n" + + " \"host\": \"testhost\",\n" + + " \"port\": 5273,\n" + + " \"region\": \"testregion\"\n" + + "}\n"; + + PasswordProvider pp = jsonMapper.readValue( + jsonMapper.writeValueAsString( + jsonMapper.readValue(jsonStr, PasswordProvider.class) + ), + PasswordProvider.class + ); + + AWSRDSTokenPasswordProvider awsPwdProvider = (AWSRDSTokenPasswordProvider) pp; + Assert.assertEquals("testuser", awsPwdProvider.getUser()); + Assert.assertEquals("testhost", awsPwdProvider.getHost()); + Assert.assertEquals(5273, awsPwdProvider.getPort()); + Assert.assertEquals("testregion", awsPwdProvider.getRegion()); + } +} diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafka/KafkaRecordEntity.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafka/KafkaRecordEntity.java new file mode 100644 index 000000000000..41c2c0a03258 --- /dev/null +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafka/KafkaRecordEntity.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.kafka; + +import org.apache.druid.data.input.InputFormat; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.indexing.kafka.KafkaRecordSupplier; +import org.apache.kafka.clients.consumer.ConsumerRecord; + +/** + * A {@link ByteEntity} generated by {@link KafkaRecordSupplier} and fed to any {@link InputFormat} used by Kafka + * indexing tasks. + *

+ * It can be used as a regular ByteEntity, in which case the Kafka record value is returned, but the {@link #getRecord} + * method also allows Kafka-aware {@link InputFormat} implementations to read the full Kafka record, including headers, + * key, and timestamp. + *

+ * NOTE: Any records with null values will be skipped, even if they contain non-null keys, or headers + *

+ * This functionality is not yet exposed through any built-in InputFormats, but is available for use in extensions. + */ +public class KafkaRecordEntity extends ByteEntity +{ + private final ConsumerRecord record; + + public KafkaRecordEntity(ConsumerRecord record) + { + super(record.value()); + this.record = record; + } + + public ConsumerRecord getRecord() + { + return record; + } +} diff --git a/extensions-core/kubernetes-extensions/pom.xml b/extensions-core/kubernetes-extensions/pom.xml new file mode 100644 index 000000000000..a22c3dd13a6f --- /dev/null +++ b/extensions-core/kubernetes-extensions/pom.xml @@ -0,0 +1,160 @@ + + + + + 4.0.0 + + org.apache.druid.extensions + druid-kubernetes-extensions + druid-kubernetes-extensions + druid-kubernetes-extensions + + + org.apache.druid + druid + 0.21.0-SNAPSHOT + ../../pom.xml + + + + 10.0.0 + + + + + org.apache.druid + druid-server + ${project.parent.version} + provided + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + org.apache.druid + druid-processing + ${project.parent.version} + provided + + + + io.kubernetes + client-java + ${kubernetes.client.version} + + + io.kubernetes + client-java-extended + ${kubernetes.client.version} + + + io.kubernetes + client-java-api + ${kubernetes.client.version} + + + + + junit + junit + test + + + org.easymock + easymock + test + + + + + org.bouncycastle + bcprov-jdk15on + 1.68 + runtime + + + + + com.google.code.findbugs + jsr305 + provided + + + com.google.inject + guice + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.google.inject.extensions + guice-multibindings + provided + + + joda-time + joda-time + provided + + + com.google.guava + guava + provided + + + com.fasterxml.jackson.core + jackson-core + provided + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.5 + + + + org/apache/druid/k8s/discovery/K8sDiscoveryModule* + + + org/apache/druid/k8s/discovery/DefaultK8sApiClient* + org/apache/druid/k8s/discovery/DefaultK8sLeaderElectorFactory* + + + + + + diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DefaultK8sApiClient.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DefaultK8sApiClient.java new file mode 100644 index 000000000000..32ad62316006 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DefaultK8sApiClient.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import com.google.inject.Inject; +import io.kubernetes.client.custom.V1Patch; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1Pod; +import io.kubernetes.client.openapi.models.V1PodList; +import io.kubernetes.client.util.Watch; +import org.apache.druid.discovery.DiscoveryDruidNode; +import org.apache.druid.discovery.NodeRole; +import org.apache.druid.guice.annotations.Json; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.logger.Logger; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.HashMap; +import java.util.Map; + +/** + * Concrete {@link K8sApiClient} impl using k8s-client java lib. + */ +public class DefaultK8sApiClient implements K8sApiClient +{ + private static final Logger LOGGER = new Logger(DefaultK8sApiClient.class); + + private final ApiClient realK8sClient; + private final CoreV1Api coreV1Api; + private final ObjectMapper jsonMapper; + + @Inject + public DefaultK8sApiClient(ApiClient realK8sClient, @Json ObjectMapper jsonMapper) + { + this.realK8sClient = realK8sClient; + this.coreV1Api = new CoreV1Api(realK8sClient); + this.jsonMapper = jsonMapper; + } + + @Override + public void patchPod(String podName, String podNamespace, String jsonPatchStr) + { + try { + coreV1Api.patchNamespacedPod(podName, podNamespace, new V1Patch(jsonPatchStr), "true", null, null, null); + } + catch (ApiException ex) { + throw new RE(ex, "Failed to patch pod[%s/%s], code[%d], error[%s].", podNamespace, podName, ex.getCode(), ex.getResponseBody()); + } + } + + @Override + public DiscoveryDruidNodeList listPods( + String podNamespace, + String labelSelector, + NodeRole nodeRole + ) + { + try { + V1PodList podList = coreV1Api.listNamespacedPod(podNamespace, null, null, null, null, labelSelector, 0, null, null, null); + Preconditions.checkState(podList != null, "WTH: NULL podList"); + + Map allNodes = new HashMap(); + for (V1Pod podDef : podList.getItems()) { + DiscoveryDruidNode node = getDiscoveryDruidNodeFromPodDef(nodeRole, podDef); + allNodes.put(node.getDruidNode().getHostAndPortToUse(), node); + } + return new DiscoveryDruidNodeList(podList.getMetadata().getResourceVersion(), allNodes); + } + catch (ApiException ex) { + throw new RE(ex, "Expection in listing pods, code[%d] and error[%s].", ex.getCode(), ex.getResponseBody()); + } + } + + private DiscoveryDruidNode getDiscoveryDruidNodeFromPodDef(NodeRole nodeRole, V1Pod podDef) + { + String jsonStr = podDef.getMetadata().getAnnotations().get(K8sDruidNodeAnnouncer.getInfoAnnotation(nodeRole)); + try { + return jsonMapper.readValue(jsonStr, DiscoveryDruidNode.class); + } + catch (JsonProcessingException ex) { + throw new RE(ex, "Failed to deserialize DiscoveryDruidNode[%s]", jsonStr); + } + } + + @Override + public WatchResult watchPods(String namespace, String labelSelector, String lastKnownResourceVersion, NodeRole nodeRole) + { + try { + Watch watch = + Watch.createWatch( + realK8sClient, + coreV1Api.listNamespacedPodCall(namespace, null, true, null, null, + labelSelector, null, lastKnownResourceVersion, 0, true, null + ), + new TypeReference>() + { + }.getType() + ); + + return new WatchResult() + { + private Watch.Response obj; + + @Override + public boolean hasNext() throws SocketTimeoutException + { + try { + while (watch.hasNext()) { + Watch.Response item = watch.next(); + if (item != null && item.type != null) { + obj = new Watch.Response( + item.type, + new DiscoveryDruidNodeAndResourceVersion( + item.object.getMetadata().getResourceVersion(), + getDiscoveryDruidNodeFromPodDef(nodeRole, item.object) + ) + ); + return true; + } else { + LOGGER.error("WTH! item or item.type is NULL"); + } + } + } + catch (RuntimeException ex) { + if (ex.getCause() instanceof SocketTimeoutException) { + throw (SocketTimeoutException) ex.getCause(); + } else { + throw ex; + } + } + + return false; + } + + @Override + public Watch.Response next() + { + return obj; + } + + @Override + public void close() + { + try { + watch.close(); + } + catch (IOException ex) { + throw new RE(ex, "Exception while closing watch."); + } + } + }; + } + catch (ApiException ex) { + if (ex.getCode() == 410) { + // k8s no longer has history that we need + return null; + } + + throw new RE(ex, "Expection in watching pods, code[%d] and error[%s].", ex.getCode(), ex.getResponseBody()); + } + } +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DefaultK8sLeaderElectorFactory.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DefaultK8sLeaderElectorFactory.java new file mode 100644 index 000000000000..7ac6b5e54ba4 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DefaultK8sLeaderElectorFactory.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.google.inject.Inject; +import io.kubernetes.client.extended.leaderelection.LeaderElectionConfig; +import io.kubernetes.client.extended.leaderelection.LeaderElector; +import io.kubernetes.client.extended.leaderelection.Lock; +import io.kubernetes.client.extended.leaderelection.resourcelock.ConfigMapLock; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.ApiException; +import org.apache.druid.java.util.common.RE; + +import java.time.Duration; + +/** + * Concrete {@link K8sLeaderElectorFactory} impl using k8s-client java lib. + */ +public class DefaultK8sLeaderElectorFactory implements K8sLeaderElectorFactory +{ + private final ApiClient realK8sClient; + private final K8sDiscoveryConfig discoveryConfig; + + @Inject + public DefaultK8sLeaderElectorFactory(ApiClient realK8sClient, K8sDiscoveryConfig discoveryConfig) + { + this.realK8sClient = realK8sClient; + this.discoveryConfig = discoveryConfig; + } + + @Override + public K8sLeaderElector create(String candidateId, String namespace, String lockResourceName) + { + Lock lock = createLock(candidateId, namespace, lockResourceName, realK8sClient); + LeaderElectionConfig leaderElectionConfig = + new LeaderElectionConfig( + lock, + Duration.ofMillis(discoveryConfig.getLeaseDuration().getMillis()), + Duration.ofMillis(discoveryConfig.getRenewDeadline().getMillis()), + Duration.ofMillis(discoveryConfig.getRetryPeriod().getMillis()) + ); + LeaderElector leaderElector = new LeaderElector(leaderElectionConfig); + + return new K8sLeaderElector() + { + @Override + public String getCurrentLeader() + { + try { + return lock.get().getHolderIdentity(); + } + catch (ApiException ex) { + throw new RE(ex, "Failed to get current leader for [%s]", lockResourceName); + } + } + + @Override + public void run(Runnable startLeadingHook, Runnable stopLeadingHook) + { + leaderElector.run(startLeadingHook, stopLeadingHook); + } + }; + } + + private Lock createLock(String candidateId, String namespace, String lockResourceName, ApiClient k8sApiClient) + { + return new ConfigMapLock( + namespace, + lockResourceName, + candidateId, + k8sApiClient + ); + } +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DiscoveryDruidNodeAndResourceVersion.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DiscoveryDruidNodeAndResourceVersion.java new file mode 100644 index 000000000000..6b634c7317cc --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DiscoveryDruidNodeAndResourceVersion.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import org.apache.druid.discovery.DiscoveryDruidNode; + +public class DiscoveryDruidNodeAndResourceVersion +{ + private final String resourceVersion; + private final DiscoveryDruidNode node; + + public DiscoveryDruidNodeAndResourceVersion(String resourceVersion, DiscoveryDruidNode node) + { + this.resourceVersion = resourceVersion; + this.node = node; + } + + public String getResourceVersion() + { + return resourceVersion; + } + + public DiscoveryDruidNode getNode() + { + return node; + } +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DiscoveryDruidNodeList.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DiscoveryDruidNodeList.java new file mode 100644 index 000000000000..cdc82ebdff68 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/DiscoveryDruidNodeList.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.google.common.base.Preconditions; +import org.apache.druid.discovery.DiscoveryDruidNode; + +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.Map; + +public class DiscoveryDruidNodeList +{ + private final String resourceVersion; + private final Map druidNodes; + + public DiscoveryDruidNodeList( + String resourceVersion, + @Nullable Map druidNodes + ) + { + this.resourceVersion = Preconditions.checkNotNull(resourceVersion, "NULL resource version!"); + this.druidNodes = druidNodes == null ? Collections.emptyMap() : druidNodes; + } + + public String getResourceVersion() + { + return resourceVersion; + } + + public Map getDruidNodes() + { + return druidNodes; + } +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sApiClient.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sApiClient.java new file mode 100644 index 000000000000..1e61677420c8 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sApiClient.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import org.apache.druid.discovery.NodeRole; + +/** + * Interface to abstract pod read/update with K8S API Server to allow unit tests with mock impl. + */ +public interface K8sApiClient +{ + void patchPod(String podName, String namespace, String jsonPatchStr); + + DiscoveryDruidNodeList listPods(String namespace, String labelSelector, NodeRole nodeRole); + + /** + * @return NULL if history not available or else return the {@link WatchResult} object + */ + WatchResult watchPods(String namespace, String labelSelector, String lastKnownResourceVersion, NodeRole nodeRole); +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDiscoveryConfig.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDiscoveryConfig.java new file mode 100644 index 000000000000..998b8641c83a --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDiscoveryConfig.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import org.apache.druid.java.util.common.logger.Logger; +import org.joda.time.Duration; + +import javax.annotation.Nonnull; +import java.util.Objects; +import java.util.regex.Pattern; + +public class K8sDiscoveryConfig +{ + private static final Logger LOGGER = new Logger(K8sDiscoveryConfig.class); + + public static final Pattern K8S_RESOURCE_NAME_REGEX = Pattern.compile("[a-z0-9][a-z0-9-]*[a-z0-9]"); + + @JsonProperty + @Nonnull + private final String clusterIdentifier; + + @JsonProperty + private final String podNameEnvKey; + + @JsonProperty + private final String podNamespaceEnvKey; + + @JsonProperty + private final String coordinatorLeaderElectionConfigMapNamespace; + + @JsonProperty + private final String overlordLeaderElectionConfigMapNamespace; + + @JsonProperty + private final Duration leaseDuration; + + @JsonProperty + private final Duration renewDeadline; + + @JsonProperty + private final Duration retryPeriod; + + @JsonCreator + public K8sDiscoveryConfig( + @JsonProperty("clusterIdentifier") String clusterIdentifier, + @JsonProperty("podNameEnvKey") String podNameEnvKey, + @JsonProperty("podNamespaceEnvKey") String podNamespaceEnvKey, + @JsonProperty("coordinatorLeaderElectionConfigMapNamespace") String coordinatorLeaderElectionConfigMapNamespace, + @JsonProperty("overlordLeaderElectionConfigMapNamespace") String overlordLeaderElectionConfigMapNamespace, + @JsonProperty("leaseDuration") Duration leaseDuration, + @JsonProperty("renewDeadline") Duration renewDeadline, + @JsonProperty("retryPeriod") Duration retryPeriod + ) + { + Preconditions.checkArgument(clusterIdentifier != null && !clusterIdentifier.isEmpty(), "null/empty clusterIdentifier"); + Preconditions.checkArgument( + K8S_RESOURCE_NAME_REGEX.matcher(clusterIdentifier).matches(), + "clusterIdentifier[%s] is used in k8s resource name and must match regex[%s]", + clusterIdentifier, + K8S_RESOURCE_NAME_REGEX.pattern() + ); + this.clusterIdentifier = clusterIdentifier; + + this.podNameEnvKey = podNameEnvKey == null ? "POD_NAME" : podNameEnvKey; + this.podNamespaceEnvKey = podNamespaceEnvKey == null ? "POD_NAMESPACE" : podNamespaceEnvKey; + + if (coordinatorLeaderElectionConfigMapNamespace == null) { + LOGGER.warn("IF coordinator pods run in multiple namespaces, then you MUST provide coordinatorLeaderElectionConfigMapNamespace"); + } + this.coordinatorLeaderElectionConfigMapNamespace = coordinatorLeaderElectionConfigMapNamespace; + + if (overlordLeaderElectionConfigMapNamespace == null) { + LOGGER.warn("IF overlord pods run in multiple namespaces, then you MUST provide overlordLeaderElectionConfigMapNamespace"); + } + this.overlordLeaderElectionConfigMapNamespace = overlordLeaderElectionConfigMapNamespace; + + this.leaseDuration = leaseDuration == null ? Duration.millis(60000) : leaseDuration; + this.renewDeadline = renewDeadline == null ? Duration.millis(17000) : renewDeadline; + this.retryPeriod = retryPeriod == null ? Duration.millis(5000) : retryPeriod; + } + + @JsonProperty + public String getClusterIdentifier() + { + return clusterIdentifier; + } + + @JsonProperty + public String getPodNameEnvKey() + { + return podNameEnvKey; + } + + @JsonProperty + public String getPodNamespaceEnvKey() + { + return podNamespaceEnvKey; + } + + @JsonProperty + public String getCoordinatorLeaderElectionConfigMapNamespace() + { + return coordinatorLeaderElectionConfigMapNamespace; + } + + @JsonProperty + public String getOverlordLeaderElectionConfigMapNamespace() + { + return overlordLeaderElectionConfigMapNamespace; + } + + @JsonProperty + public Duration getLeaseDuration() + { + return leaseDuration; + } + + @JsonProperty + public Duration getRenewDeadline() + { + return renewDeadline; + } + + @JsonProperty + public Duration getRetryPeriod() + { + return retryPeriod; + } + + @Override + public String toString() + { + return "K8sDiscoveryConfig{" + + "clusterIdentifier='" + clusterIdentifier + '\'' + + ", podNameEnvKey='" + podNameEnvKey + '\'' + + ", podNamespaceEnvKey='" + podNamespaceEnvKey + '\'' + + ", coordinatorLeaderElectionConfigMapNamespace='" + coordinatorLeaderElectionConfigMapNamespace + '\'' + + ", overlordLeaderElectionConfigMapNamespace='" + overlordLeaderElectionConfigMapNamespace + '\'' + + ", leaseDuration=" + leaseDuration + + ", renewDeadline=" + renewDeadline + + ", retryPeriod=" + retryPeriod + + '}'; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + K8sDiscoveryConfig that = (K8sDiscoveryConfig) o; + return clusterIdentifier.equals(that.clusterIdentifier) && + Objects.equals(podNameEnvKey, that.podNameEnvKey) && + Objects.equals(podNamespaceEnvKey, that.podNamespaceEnvKey) && + Objects.equals( + coordinatorLeaderElectionConfigMapNamespace, + that.coordinatorLeaderElectionConfigMapNamespace + ) && + Objects.equals( + overlordLeaderElectionConfigMapNamespace, + that.overlordLeaderElectionConfigMapNamespace + ) && + Objects.equals(leaseDuration, that.leaseDuration) && + Objects.equals(renewDeadline, that.renewDeadline) && + Objects.equals(retryPeriod, that.retryPeriod); + } + + @Override + public int hashCode() + { + return Objects.hash( + clusterIdentifier, + podNameEnvKey, + podNamespaceEnvKey, + coordinatorLeaderElectionConfigMapNamespace, + overlordLeaderElectionConfigMapNamespace, + leaseDuration, + renewDeadline, + retryPeriod + ); + } +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDiscoveryModule.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDiscoveryModule.java new file mode 100644 index 000000000000..6da6819ff4b5 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDiscoveryModule.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.fasterxml.jackson.databind.Module; +import com.google.inject.Binder; +import com.google.inject.Inject; +import com.google.inject.Key; +import com.google.inject.Provider; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.util.Config; +import org.apache.druid.client.coordinator.Coordinator; +import org.apache.druid.client.indexing.IndexingService; +import org.apache.druid.discovery.DruidLeaderSelector; +import org.apache.druid.discovery.DruidNodeAnnouncer; +import org.apache.druid.discovery.DruidNodeDiscoveryProvider; +import org.apache.druid.guice.JsonConfigProvider; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.guice.PolyBind; +import org.apache.druid.guice.annotations.Self; +import org.apache.druid.initialization.DruidModule; +import org.apache.druid.server.DruidNode; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +public class K8sDiscoveryModule implements DruidModule +{ + private static final String K8S_KEY = "k8s"; + + @Override + public List getJacksonModules() + { + return Collections.emptyList(); + } + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.discovery.k8s", K8sDiscoveryConfig.class); + + binder.bind(ApiClient.class) + .toProvider( + () -> { + try { + // Note: we can probably improve things here about figuring out how to find the K8S API server, + // HTTP client timeouts etc. + return Config.defaultClient(); + } + catch (IOException ex) { + throw new RuntimeException("Failed to create K8s ApiClient instance", ex); + } + } + ) + .in(LazySingleton.class); + + binder.bind(K8sApiClient.class).to(DefaultK8sApiClient.class).in(LazySingleton.class); + binder.bind(K8sLeaderElectorFactory.class).to(DefaultK8sLeaderElectorFactory.class).in(LazySingleton.class); + + PolyBind.optionBinder(binder, Key.get(DruidNodeDiscoveryProvider.class)) + .addBinding(K8S_KEY) + .to(K8sDruidNodeDiscoveryProvider.class) + .in(LazySingleton.class); + + PolyBind.optionBinder(binder, Key.get(DruidNodeAnnouncer.class)) + .addBinding(K8S_KEY) + .to(K8sDruidNodeAnnouncer.class) + .in(LazySingleton.class); + + PolyBind.optionBinder(binder, Key.get(DruidLeaderSelector.class, Coordinator.class)) + .addBinding(K8S_KEY) + .toProvider( + new DruidLeaderSelectorProvider(true) + ) + .in(LazySingleton.class); + + PolyBind.optionBinder(binder, Key.get(DruidLeaderSelector.class, IndexingService.class)) + .addBinding(K8S_KEY) + .toProvider( + new DruidLeaderSelectorProvider(false) + ) + .in(LazySingleton.class); + } + + private static class DruidLeaderSelectorProvider implements Provider + { + @Inject + @Self + private DruidNode druidNode; + + @Inject + private PodInfo podInfo; + + @Inject + private K8sDiscoveryConfig discoveryConfig; + + @Inject + private Provider k8sApiClientProvider; + + private boolean isCoordinator; + + DruidLeaderSelectorProvider(boolean isCoordinator) + { + this.isCoordinator = isCoordinator; + } + + @Override + public DruidLeaderSelector get() + { + // Note: these can not be setup in the constructor because injected K8sDiscoveryConfig and PodInfo + // are not available at that time. + String lockResourceName; + String lockResourceNamespace; + + if (isCoordinator) { + lockResourceName = discoveryConfig.getClusterIdentifier() + "-leaderelection-coordinator"; + lockResourceNamespace = discoveryConfig.getCoordinatorLeaderElectionConfigMapNamespace() == null ? + podInfo.getPodNamespace() : discoveryConfig.getCoordinatorLeaderElectionConfigMapNamespace(); + } else { + lockResourceName = discoveryConfig.getClusterIdentifier() + "-leaderelection-overlord"; + lockResourceNamespace = discoveryConfig.getOverlordLeaderElectionConfigMapNamespace() == null ? + podInfo.getPodNamespace() : discoveryConfig.getOverlordLeaderElectionConfigMapNamespace(); + } + + return new K8sDruidLeaderSelector( + druidNode, + lockResourceName, + lockResourceNamespace, + discoveryConfig, + new DefaultK8sLeaderElectorFactory(k8sApiClientProvider.get(), discoveryConfig) + ); + } + } +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidLeaderSelector.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidLeaderSelector.java new file mode 100644 index 000000000000..2cf5a8d581b2 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidLeaderSelector.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.google.common.base.Preconditions; +import org.apache.druid.annotations.SuppressFBWarnings; +import org.apache.druid.concurrent.LifecycleLock; +import org.apache.druid.discovery.DruidLeaderSelector; +import org.apache.druid.guice.annotations.Self; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.guava.CloseQuietly; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.server.DruidNode; + +import javax.annotation.Nullable; + +public class K8sDruidLeaderSelector implements DruidLeaderSelector +{ + private static final EmittingLogger LOGGER = new EmittingLogger(K8sDruidLeaderSelector.class); + + private final LifecycleLock lifecycleLock = new LifecycleLock(); + + private DruidLeaderSelector.Listener listener = null; + private final LeaderElectorAsyncWrapper leaderLatch; + + private volatile boolean leader = false; + + @SuppressFBWarnings(value = "VO_VOLATILE_INCREMENT", justification = "incremented but in single thread") + private volatile int term = 0; + + public K8sDruidLeaderSelector(@Self DruidNode self, String lockResourceName, String lockResourceNamespace, K8sDiscoveryConfig discoveryConfig, K8sLeaderElectorFactory k8sLeaderElectorFactory) + { + this.leaderLatch = new LeaderElectorAsyncWrapper( + self.getServiceScheme() + "://" + self.getHostAndPortToUse(), + lockResourceName, + lockResourceNamespace, + discoveryConfig, + k8sLeaderElectorFactory + ); + } + + private void startLeaderElector(LeaderElectorAsyncWrapper leaderElector) + { + leaderElector.run( + () -> { + try { + if (leader) { + LOGGER.warn("I'm being asked to become leader. But I am already the leader. Ignored event."); + return; + } + + leader = true; + term++; + listener.becomeLeader(); + } + catch (Throwable ex) { + LOGGER.makeAlert(ex, "listener becomeLeader() failed. Unable to become leader").emit(); + + CloseQuietly.close(leaderLatch); + leader = false; + //Exit and Kubernetes would simply create a new replacement pod. + System.exit(1); + } + }, + () -> { + try { + if (!leader) { + LOGGER.warn("I'm being asked to stop being leader. But I am not the leader. Ignored event."); + return; + } + + leader = false; + listener.stopBeingLeader(); + } + catch (Throwable ex) { + LOGGER.makeAlert(ex, "listener.stopBeingLeader() failed. Unable to stopBeingLeader").emit(); + } + } + ); + } + + @Nullable + @Override + public String getCurrentLeader() + { + try { + return leaderLatch.getCurrentLeader(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean isLeader() + { + return leader; + } + + @Override + public int localTerm() + { + return term; + } + + @Override + public void registerListener(DruidLeaderSelector.Listener listener) + { + Preconditions.checkArgument(listener != null, "listener is null."); + + if (!lifecycleLock.canStart()) { + throw new ISE("can't start."); + } + try { + this.listener = listener; + startLeaderElector(leaderLatch); + lifecycleLock.started(); + } + catch (Exception ex) { + throw new RuntimeException(ex); + } + finally { + lifecycleLock.exitStart(); + } + } + + @Override + public void unregisterListener() + { + if (!lifecycleLock.canStop()) { + throw new ISE("can't stop."); + } + CloseQuietly.close(leaderLatch); + } +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java new file mode 100644 index 000000000000..29c06f443a3e --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncer.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; +import org.apache.druid.discovery.DiscoveryDruidNode; +import org.apache.druid.discovery.DruidNodeAnnouncer; +import org.apache.druid.discovery.NodeRole; +import org.apache.druid.guice.annotations.Json; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.RetryUtils; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.server.DruidNode; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Announcement creates following in the pod def... + * + * Labels - + * druidDiscoveryAnnouncement- = true + * druidDiscoveryAnnouncement-id = encodeHostPort(host:port) + * druidDiscoveryAnnouncement-cluster-identifier = + * + * Annotation - + * druidNodeInfo- = json_serialize(DiscoveryDruidNode) + * + * Note that, a node can have multiple roles e.g. coordinator can take up overlord's role as well. + */ +public class K8sDruidNodeAnnouncer implements DruidNodeAnnouncer +{ + private static final Logger LOGGER = new Logger(K8sDruidNodeAnnouncer.class); + + private static String POD_LABELS_PATH_PREFIX = "/metadata/labels"; + private static String POD_ANNOTATIONS_PATH_PREFIX = "/metadata/annotations"; + + private static final String OP_ADD = "add"; + private static final String OP_REMOVE = "remove"; + + public static final String ANNOUNCEMENT_DONE = "true"; + + private final ObjectMapper jsonMapper; + private final K8sDiscoveryConfig discoveryConfig; + private final PodInfo podInfo; + private final K8sApiClient k8sApiClient; + + @Inject + public K8sDruidNodeAnnouncer( + PodInfo podInfo, + K8sDiscoveryConfig discoveryConfig, + K8sApiClient k8sApiClient, + @Json ObjectMapper jsonMapper + ) + { + this.discoveryConfig = discoveryConfig; + this.podInfo = podInfo; + this.k8sApiClient = k8sApiClient; + this.jsonMapper = jsonMapper; + } + + @Override + public void announce(DiscoveryDruidNode discoveryDruidNode) + { + LOGGER.info("Announcing DiscoveryDruidNode[%s]", discoveryDruidNode); + + String roleAnnouncementLabel = getRoleAnnouncementLabel(discoveryDruidNode.getNodeRole()); + String idAnnouncementLabel = getIdAnnouncementLabel(); + String clusterIdentifierAnnouncementLabel = getClusterIdentifierAnnouncementLabel(); + String infoAnnotation = getInfoAnnotation(discoveryDruidNode.getNodeRole()); + + try { + List> patches = new ArrayList<>(); + + // Note: We assume here that at least one label and annotation exists on the pod already, so that + // paths where labels/annotations are created, pre-exist. + // See https://github.com/kubernetes-sigs/kustomize/issues/2986 , we can add workaround of getting pod spec, + // checking if label/annotation path exists and create if not, however that could lead to race conditions + // so assuming the existence for now. + patches.add(createPatchObj(OP_ADD, getPodDefLabelPath(roleAnnouncementLabel), ANNOUNCEMENT_DONE)); + patches.add(createPatchObj(OP_ADD, getPodDefLabelPath(idAnnouncementLabel), encodeHostPort(discoveryDruidNode.getDruidNode().getHostAndPortToUse()))); + patches.add(createPatchObj(OP_ADD, getPodDefLabelPath(clusterIdentifierAnnouncementLabel), discoveryConfig.getClusterIdentifier())); + patches.add(createPatchObj(OP_ADD, getPodDefAnnocationPath(infoAnnotation), jsonMapper.writeValueAsString(discoveryDruidNode))); + + // Creating patch string outside of retry block to not retry json serialization failures + String jsonPatchStr = jsonMapper.writeValueAsString(patches); + LOGGER.info("Json Patch For Node Announcement: [%s]", jsonPatchStr); + + RetryUtils.retry( + () -> { + k8sApiClient.patchPod(podInfo.getPodName(), podInfo.getPodNamespace(), jsonPatchStr); + return "na"; + }, + (throwable) -> true, + 3 + ); + + LOGGER.info("Announced DiscoveryDruidNode[%s]", discoveryDruidNode); + } + catch (Exception ex) { + throw new RE(ex, "Failed to announce DiscoveryDruidNode[%s]", discoveryDruidNode); + } + } + + @Override + public void unannounce(DiscoveryDruidNode discoveryDruidNode) + { + LOGGER.info("Unannouncing DiscoveryDruidNode[%s]", discoveryDruidNode); + + String roleAnnouncementLabel = getRoleAnnouncementLabel(discoveryDruidNode.getNodeRole()); + String idAnnouncementLabel = getIdAnnouncementLabel(); + String clusterIdentifierAnnouncementLabel = getClusterIdentifierAnnouncementLabel(); + String infoAnnotation = getInfoAnnotation(discoveryDruidNode.getNodeRole()); + + try { + List> patches = new ArrayList<>(); + patches.add(createPatchObj(OP_REMOVE, getPodDefLabelPath(roleAnnouncementLabel), null)); + patches.add(createPatchObj(OP_REMOVE, getPodDefLabelPath(idAnnouncementLabel), null)); + patches.add(createPatchObj(OP_REMOVE, getPodDefLabelPath(clusterIdentifierAnnouncementLabel), null)); + patches.add(createPatchObj(OP_REMOVE, getPodDefAnnocationPath(infoAnnotation), null)); + + // Creating patch string outside of retry block to not retry json serialization failures + String jsonPatchStr = jsonMapper.writeValueAsString(patches); + + RetryUtils.retry( + () -> { + k8sApiClient.patchPod(podInfo.getPodName(), podInfo.getPodNamespace(), jsonPatchStr); + return "na"; + }, + (throwable) -> true, + 3 + ); + + LOGGER.info("Unannounced DiscoveryDruidNode[%s]", discoveryDruidNode); + + } + catch (Exception ex) { + // Unannouncement happens when druid process is shutting down, there is no point throwing exception + // in shutdown sequence. + if (ex instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + + LOGGER.error(ex, "Failed to unannounce DiscoveryDruidNode[%s]", discoveryDruidNode); + } + } + + private Map createPatchObj(String op, String path, Object value) + { + if (value == null) { + return ImmutableMap.of( + "op", op, + "path", path + ); + } else { + return ImmutableMap.of( + "op", op, + "path", path, + "value", value + ); + } + } + + public static String getRoleAnnouncementLabel(NodeRole nodeRole) + { + return StringUtils.format("druidDiscoveryAnnouncement-%s", nodeRole.getJsonName()); + } + + private static String getIdAnnouncementLabel() + { + return "druidDiscoveryAnnouncement-id"; + } + + public static String getClusterIdentifierAnnouncementLabel() + { + return "druidDiscoveryAnnouncement-cluster-identifier"; + } + + public static String getInfoAnnotation(NodeRole nodeRole) + { + return StringUtils.format("druidNodeInfo-%s", nodeRole.getJsonName()); + } + + public static String getLabelSelectorForNodeRole(K8sDiscoveryConfig discoveryConfig, NodeRole nodeRole) + { + return StringUtils.format( + "%s=%s,%s=%s", + getClusterIdentifierAnnouncementLabel(), + discoveryConfig.getClusterIdentifier(), + K8sDruidNodeAnnouncer.getRoleAnnouncementLabel(nodeRole), + K8sDruidNodeAnnouncer.ANNOUNCEMENT_DONE + ); + } + + public static String getLabelSelectorForNode(K8sDiscoveryConfig discoveryConfig, NodeRole nodeRole, DruidNode node) + { + return StringUtils.format( + "%s=%s,%s=%s,%s=%s", + getClusterIdentifierAnnouncementLabel(), + discoveryConfig.getClusterIdentifier(), + K8sDruidNodeAnnouncer.getRoleAnnouncementLabel(nodeRole), + K8sDruidNodeAnnouncer.ANNOUNCEMENT_DONE, + K8sDruidNodeAnnouncer.getIdAnnouncementLabel(), + encodeHostPort(node.getHostAndPortToUse()) + ); + } + + private String getPodDefLabelPath(String label) + { + return StringUtils.format("%s/%s", POD_LABELS_PATH_PREFIX, label); + } + + private String getPodDefAnnocationPath(String annotation) + { + return StringUtils.format("%s/%s", POD_ANNOTATIONS_PATH_PREFIX, annotation); + } + + private static String encodeHostPort(String hostPort) + { + //K8S requires that label values must match regex (([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])? + //So, it is essential to replace ':' with '-' + + // it is assumed that hostname does not have ':' in it except for separating host and port + Preconditions.checkState( + hostPort.indexOf(':') == hostPort.lastIndexOf(':'), + "hostname in host:port[%s] has ':' in it", hostPort + ); + + return hostPort.replace(':', '-'); + } + + private String replaceLast(String str, char oldChar, char newChar) + { + char[] chars = str.toCharArray(); + for (int i = chars.length - 1; i >= 0; i--) { + if (chars[i] == oldChar) { + chars[i] = newChar; + break; + } + } + + return String.valueOf(chars); + } +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeDiscoveryProvider.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeDiscoveryProvider.java new file mode 100644 index 000000000000..fd26c1e0fcf3 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDruidNodeDiscoveryProvider.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.inject.Inject; +import io.kubernetes.client.util.Watch; +import org.apache.druid.concurrent.LifecycleLock; +import org.apache.druid.discovery.BaseNodeRoleWatcher; +import org.apache.druid.discovery.DiscoveryDruidNode; +import org.apache.druid.discovery.DruidNodeDiscovery; +import org.apache.druid.discovery.DruidNodeDiscoveryProvider; +import org.apache.druid.discovery.NodeRole; +import org.apache.druid.guice.ManageLifecycle; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.guava.CloseQuietly; +import org.apache.druid.java.util.common.lifecycle.LifecycleStart; +import org.apache.druid.java.util.common.lifecycle.LifecycleStop; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.server.DruidNode; + +import java.io.Closeable; +import java.net.SocketTimeoutException; +import java.util.Collection; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; + +@ManageLifecycle +public class K8sDruidNodeDiscoveryProvider extends DruidNodeDiscoveryProvider +{ + private static final Logger LOGGER = new Logger(K8sDruidNodeDiscoveryProvider.class); + + private final PodInfo podInfo; + private final K8sDiscoveryConfig discoveryConfig; + + private final K8sApiClient k8sApiClient; + + private ExecutorService listenerExecutor; + + private final ConcurrentHashMap nodeTypeWatchers = new ConcurrentHashMap<>(); + + private final LifecycleLock lifecycleLock = new LifecycleLock(); + + private final long watcherErrorRetryWaitMS; + + @Inject + public K8sDruidNodeDiscoveryProvider( + PodInfo podInfo, + K8sDiscoveryConfig discoveryConfig, + K8sApiClient k8sApiClient + ) + { + // at some point, if needed, watcherErrorRetryWaitMS here can be made configurable and maybe some randomization + // component as well. + this(podInfo, discoveryConfig, k8sApiClient, 10_000); + } + + @VisibleForTesting + K8sDruidNodeDiscoveryProvider( + PodInfo podInfo, + K8sDiscoveryConfig discoveryConfig, + K8sApiClient k8sApiClient, + long watcherErrorRetryWaitMS + ) + { + this.podInfo = podInfo; + this.discoveryConfig = discoveryConfig; + this.k8sApiClient = k8sApiClient; + this.watcherErrorRetryWaitMS = watcherErrorRetryWaitMS; + } + + @Override + public BooleanSupplier getForNode(DruidNode node, NodeRole nodeRole) + { + return () -> !k8sApiClient.listPods( + podInfo.getPodNamespace(), + K8sDruidNodeAnnouncer.getLabelSelectorForNode(discoveryConfig, nodeRole, node), + nodeRole + ).getDruidNodes().isEmpty(); + } + + @Override + public DruidNodeDiscovery getForNodeRole(NodeRole nodeType) + { + return getForNodeRole(nodeType, true); + } + + @VisibleForTesting + NodeRoleWatcher getForNodeRole(NodeRole nodeType, boolean startAfterCreation) + { + Preconditions.checkState(lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)); + + return nodeTypeWatchers.computeIfAbsent( + nodeType, + nType -> { + LOGGER.info("Creating NodeRoleWatcher for nodeRole [%s].", nType); + NodeRoleWatcher nodeRoleWatcher = new NodeRoleWatcher( + listenerExecutor, + nType, + podInfo, + discoveryConfig, + k8sApiClient, + watcherErrorRetryWaitMS + ); + if (startAfterCreation) { + nodeRoleWatcher.start(); + } + LOGGER.info("Created NodeRoleWatcher for nodeRole [%s].", nType); + return nodeRoleWatcher; + } + ); + } + + @LifecycleStart + public void start() + { + if (!lifecycleLock.canStart()) { + throw new ISE("can't start."); + } + + try { + LOGGER.info("starting"); + + // This is single-threaded to ensure that all listener calls are executed precisely in the oder of add/remove + // event occurences. + listenerExecutor = Execs.singleThreaded("K8sDruidNodeDiscoveryProvider-ListenerExecutor"); + + LOGGER.info("started"); + + lifecycleLock.started(); + } + finally { + lifecycleLock.exitStart(); + } + } + + @LifecycleStop + public void stop() + { + if (!lifecycleLock.canStop()) { + throw new ISE("can't stop."); + } + + LOGGER.info("stopping"); + + for (NodeRoleWatcher watcher : nodeTypeWatchers.values()) { + watcher.stop(); + } + listenerExecutor.shutdownNow(); + + LOGGER.info("stopped"); + } + + @VisibleForTesting + static class NodeRoleWatcher implements DruidNodeDiscovery + { + private static final Logger LOGGER = new Logger(NodeRoleWatcher.class); + + private final PodInfo podInfo; + private final K8sDiscoveryConfig discoveryConfig; + + private final K8sApiClient k8sApiClient; + + private ExecutorService watchExecutor; + + private final LifecycleLock lifecycleLock = new LifecycleLock(); + + private final AtomicReference watchRef = new AtomicReference<>(); + private static final Closeable STOP_MARKER = () -> {}; + + private final NodeRole nodeRole; + private final BaseNodeRoleWatcher baseNodeRoleWatcher; + + private final long watcherErrorRetryWaitMS; + + NodeRoleWatcher( + ExecutorService listenerExecutor, + NodeRole nodeRole, + PodInfo podInfo, + K8sDiscoveryConfig discoveryConfig, + K8sApiClient k8sApiClient, + long watcherErrorRetryWaitMS + ) + { + this.podInfo = podInfo; + this.discoveryConfig = discoveryConfig; + this.k8sApiClient = k8sApiClient; + + this.nodeRole = nodeRole; + this.baseNodeRoleWatcher = new BaseNodeRoleWatcher(listenerExecutor, nodeRole); + + this.watcherErrorRetryWaitMS = watcherErrorRetryWaitMS; + } + + private void watch() + { + String labelSelector = K8sDruidNodeAnnouncer.getLabelSelectorForNodeRole(discoveryConfig, nodeRole); + boolean cacheInitialized = false; + + while (lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)) { + try { + DiscoveryDruidNodeList list = k8sApiClient.listPods(podInfo.getPodNamespace(), labelSelector, nodeRole); + baseNodeRoleWatcher.resetNodes(list.getDruidNodes()); + + if (!cacheInitialized) { + baseNodeRoleWatcher.cacheInitialized(); + cacheInitialized = true; + } + + keepWatching( + podInfo.getPodNamespace(), + labelSelector, + list.getResourceVersion() + ); + } + catch (Throwable ex) { + LOGGER.error(ex, "Expection while watching for NodeRole [%s].", nodeRole); + + // Wait a little before trying again. + sleep(watcherErrorRetryWaitMS); + } + } + + LOGGER.info("Exited Watch for NodeRole [%s].", nodeRole); + } + + private void keepWatching(String namespace, String labelSelector, String resourceVersion) + { + String nextResourceVersion = resourceVersion; + while (lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)) { + try { + WatchResult iter = + k8sApiClient.watchPods(podInfo.getPodNamespace(), labelSelector, nextResourceVersion, nodeRole); + + if (iter == null) { + // history not available, we need to start from scratch + return; + } + + try { + while (iter.hasNext()) { + Watch.Response item = iter.next(); + if (item != null && item.type != null) { + switch (item.type) { + case WatchResult.ADDED: + baseNodeRoleWatcher.childAdded(item.object.getNode()); + break; + case WatchResult.DELETED: + baseNodeRoleWatcher.childRemoved(item.object.getNode()); + break; + default: + } + + // This should be updated after the action has been dealt with successfully + nextResourceVersion = item.object.getResourceVersion(); + + } else { + LOGGER.error("WTH! item or item.type is NULL"); + } + } + } + finally { + iter.close(); + } + + } + catch (SocketTimeoutException ex) { + // socket read timeout can happen normally due to k8s not having anything new to push leading to socket + // read timeout, so no error log + sleep(watcherErrorRetryWaitMS); + } + catch (Throwable ex) { + LOGGER.error(ex, "Error while watching node type [%s]", this.nodeRole); + sleep(watcherErrorRetryWaitMS); + } + } + } + + private void sleep(long ms) + { + try { + Thread.sleep(ms); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + public void start() + { + if (!lifecycleLock.canStart()) { + throw new ISE("can't start."); + } + + try { + LOGGER.info("Starting NodeRoleWatcher for [%s]...", nodeRole); + this.watchExecutor = Execs.singleThreaded(this.getClass().getName() + nodeRole.getJsonName()); + watchExecutor.submit(this::watch); + lifecycleLock.started(); + LOGGER.info("Started NodeRoleWatcher for [%s].", nodeRole); + } + finally { + lifecycleLock.exitStart(); + } + + } + + public void stop() + { + if (!lifecycleLock.canStop()) { + throw new ISE("can't stop."); + } + + try { + LOGGER.info("Stopping NodeRoleWatcher for [%s]...", nodeRole); + CloseQuietly.close(watchRef.getAndSet(STOP_MARKER)); + watchExecutor.shutdownNow(); + + if (!watchExecutor.awaitTermination(15, TimeUnit.SECONDS)) { + LOGGER.warn("Failed to stop watchExecutor for NodeRoleWatcher[%s]", nodeRole); + } + LOGGER.info("Stopped NodeRoleWatcher for [%s].", nodeRole); + } + catch (Exception ex) { + LOGGER.error(ex, "Failed to stop NodeRoleWatcher for [%s].", nodeRole); + } + } + + @Override + public Collection getAllNodes() + { + return baseNodeRoleWatcher.getAllNodes(); + } + + @Override + public void registerListener(Listener listener) + { + baseNodeRoleWatcher.registerListener(listener); + } + } +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sLeaderElectorFactory.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sLeaderElectorFactory.java new file mode 100644 index 000000000000..10bab39f65c1 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sLeaderElectorFactory.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import io.kubernetes.client.extended.leaderelection.LeaderElector; + +/** + * Interface to abstract creation/use of {@link LeaderElector} from k8s-client java lib to allow unit tests with + * mock impl. + */ +public interface K8sLeaderElectorFactory +{ + K8sLeaderElector create( + String candidateId, + String namespace, + String lockResourceName + ); +} + +interface K8sLeaderElector +{ + String getCurrentLeader(); + void run(Runnable startLeadingHook, Runnable stopLeadingHook); +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/LeaderElectorAsyncWrapper.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/LeaderElectorAsyncWrapper.java new file mode 100644 index 000000000000..f7e1b481d044 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/LeaderElectorAsyncWrapper.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.google.common.base.Preconditions; +import org.apache.druid.concurrent.LifecycleLock; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.logger.Logger; + +import java.io.Closeable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +public class LeaderElectorAsyncWrapper implements Closeable +{ + private static final Logger LOGGER = new Logger(LeaderElectorAsyncWrapper.class); + + private ExecutorService executor; + private final AtomicReference futureRef = new AtomicReference<>(); + + private final K8sLeaderElector k8sLeaderElector; + + private final LifecycleLock lifecycleLock = new LifecycleLock(); + + public LeaderElectorAsyncWrapper( + String candidateId, + String lockResourceName, + String lockResourceNamespace, + K8sDiscoveryConfig discoveryConfig, + K8sLeaderElectorFactory k8sLeaderElectorFactory + ) + { + Preconditions.checkArgument( + K8sDiscoveryConfig.K8S_RESOURCE_NAME_REGEX.matcher(lockResourceName).matches(), + "lockResourceName[%s] must match regex[%s]", + lockResourceName, + K8sDiscoveryConfig.K8S_RESOURCE_NAME_REGEX.pattern() + ); + LOGGER.info( + "Creating LeaderElector with candidateId[%s], lockResourceName[%s], k8sNamespace[%s].", + candidateId, + lockResourceName, + lockResourceNamespace + ); + + k8sLeaderElector = k8sLeaderElectorFactory.create(candidateId, lockResourceNamespace, lockResourceName); + } + + public void run(Runnable startLeadingHook, Runnable stopLeadingHook) + { + if (!lifecycleLock.canStart()) { + throw new ISE("can't start."); + } + + try { + executor = Execs.singleThreaded(this.getClass().getSimpleName()); + futureRef.set(executor.submit( + () -> { + while (lifecycleLock.awaitStarted(1, TimeUnit.MILLISECONDS)) { + try { + k8sLeaderElector.run(startLeadingHook, stopLeadingHook); + } + catch (Throwable ex) { + LOGGER.error(ex, "Exception in K8s LeaderElector.run()"); + } + } + } + )); + lifecycleLock.started(); + } + finally { + lifecycleLock.exitStart(); + } + } + + @Override + public void close() + { + if (!lifecycleLock.canStop()) { + throw new ISE("can't stop."); + } + + try { + futureRef.get().cancel(true); + executor.shutdownNow(); + if (!executor.awaitTermination(10, TimeUnit.SECONDS)) { + LOGGER.warn("Failed to terminate [%s] executor.", this.getClass().getSimpleName()); + } + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + + public String getCurrentLeader() + { + return k8sLeaderElector.getCurrentLeader(); + } +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/PodInfo.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/PodInfo.java new file mode 100644 index 000000000000..35184a3fbaa5 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/PodInfo.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.inject.Inject; +import org.apache.druid.guice.LazySingleton; + +@LazySingleton +public class PodInfo +{ + private final String podName; + private final String podNamespace; + + @Inject + public PodInfo(K8sDiscoveryConfig discoveryConfig) + { + this.podName = System.getenv(discoveryConfig.getPodNameEnvKey()); + Preconditions.checkState(podName != null && !podName.isEmpty(), "Failed to find podName"); + + this.podNamespace = System.getenv(discoveryConfig.getPodNamespaceEnvKey()); + Preconditions.checkState(podNamespace != null && !podNamespace.isEmpty(), "Failed to find podNamespace"); + } + + @VisibleForTesting + public PodInfo(String podName, String podNamespace) + { + this.podName = podName; + this.podNamespace = podNamespace; + } + + public String getPodName() + { + return podName; + } + + public String getPodNamespace() + { + return podNamespace; + } +} diff --git a/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/WatchResult.java b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/WatchResult.java new file mode 100644 index 000000000000..5f9e4178eb18 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/WatchResult.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import io.kubernetes.client.util.Watch; + +import java.net.SocketTimeoutException; + +public interface WatchResult +{ + String ADDED = "ADDED"; + String DELETED = "DELETED"; + + boolean hasNext() throws SocketTimeoutException; + + Watch.Response next(); + + void close(); +} diff --git a/extensions-core/kubernetes-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-core/kubernetes-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..f1461fb8dd54 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.k8s.discovery.K8sDiscoveryModule diff --git a/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sAnnouncerAndDiscoveryIntTest.java b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sAnnouncerAndDiscoveryIntTest.java new file mode 100644 index 000000000000..ab13b356382b --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sAnnouncerAndDiscoveryIntTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.kubernetes.client.util.Config; +import org.apache.druid.discovery.DiscoveryDruidNode; +import org.apache.druid.discovery.DruidNodeDiscovery; +import org.apache.druid.discovery.NodeRole; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.server.DruidNode; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.Collection; +import java.util.Iterator; +import java.util.concurrent.CountDownLatch; +import java.util.function.BooleanSupplier; + +/** + * This is not a UT, but very helpful when making changes to ensure things work with real K8S Api Server. + * It is ignored in the build but checked in the reporitory for running manually by devs. + */ +@Ignore("Needs K8S API Server") +public class K8sAnnouncerAndDiscoveryIntTest +{ + private final DiscoveryDruidNode testNode = new DiscoveryDruidNode( + new DruidNode("druid/router", "test-host", true, 80, null, true, false), + NodeRole.ROUTER, + null + ); + + private final ObjectMapper jsonMapper = new DefaultObjectMapper(); + + private final PodInfo podInfo = new PodInfo("busybox", "default"); + + private final K8sDiscoveryConfig discoveryConfig = new K8sDiscoveryConfig("druid-cluster", null, null, null, null, null, null, null); + + @Test(timeout = 30000L) + public void testAnnouncementAndDiscoveryWorkflow() throws Exception + { + K8sApiClient k8sApiClient = new DefaultK8sApiClient(Config.defaultClient(), new DefaultObjectMapper()); + + K8sDruidNodeDiscoveryProvider discoveryProvider = new K8sDruidNodeDiscoveryProvider( + podInfo, + discoveryConfig, + k8sApiClient + ); + discoveryProvider.start(); + + BooleanSupplier nodeInquirer = discoveryProvider.getForNode(testNode.getDruidNode(), NodeRole.ROUTER); + Assert.assertFalse(nodeInquirer.getAsBoolean()); + + DruidNodeDiscovery discovery = discoveryProvider.getForNodeRole(NodeRole.ROUTER); + + CountDownLatch nodeViewInitialized = new CountDownLatch(1); + CountDownLatch nodeAppeared = new CountDownLatch(1); + CountDownLatch nodeDisappeared = new CountDownLatch(1); + + discovery.registerListener( + new DruidNodeDiscovery.Listener() + { + @Override + public void nodesAdded(Collection nodes) + { + Iterator iter = nodes.iterator(); + if (iter.hasNext() && testNode.getDruidNode().getHostAndPort().equals(iter.next().getDruidNode().getHostAndPort())) { + nodeAppeared.countDown(); + } + } + + @Override + public void nodesRemoved(Collection nodes) + { + Iterator iter = nodes.iterator(); + if (iter.hasNext() && testNode.getDruidNode().getHostAndPort().equals(iter.next().getDruidNode().getHostAndPort())) { + nodeDisappeared.countDown(); + } + } + + @Override + public void nodeViewInitialized() + { + nodeViewInitialized.countDown(); + } + } + ); + + nodeViewInitialized.await(); + + K8sDruidNodeAnnouncer announcer = new K8sDruidNodeAnnouncer(podInfo, discoveryConfig, k8sApiClient, jsonMapper); + announcer.announce(testNode); + + nodeAppeared.await(); + + Assert.assertTrue(nodeInquirer.getAsBoolean()); + + announcer.unannounce(testNode); + + nodeDisappeared.await(); + + Assert.assertFalse(nodeInquirer.getAsBoolean()); + + discoveryProvider.stop(); + } +} diff --git a/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDiscoveryConfigTest.java b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDiscoveryConfigTest.java new file mode 100644 index 000000000000..b76ae4b62462 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDiscoveryConfigTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.joda.time.Duration; +import org.junit.Assert; +import org.junit.Test; + +public class K8sDiscoveryConfigTest +{ + private final ObjectMapper jsonMapper = new DefaultObjectMapper(); + + @Test + public void testDefaultValuesSerde() throws Exception + { + testSerde( + "{\"clusterIdentifier\": \"test-cluster\"}\n", + new K8sDiscoveryConfig("test-cluster", null, null, null, null, null, null, null) + ); + } + + @Test + public void testCustomizedValuesSerde() throws Exception + { + testSerde( + "{\n" + + " \"clusterIdentifier\": \"test-cluster\",\n" + + " \"podNameEnvKey\": \"PODNAMETEST\",\n" + + " \"podNamespaceEnvKey\": \"PODNAMESPACETEST\",\n" + + " \"coordinatorLeaderElectionConfigMapNamespace\": \"coordinatorns\",\n" + + " \"overlordLeaderElectionConfigMapNamespace\": \"overlordns\",\n" + + " \"leaseDuration\": \"PT3S\",\n" + + " \"renewDeadline\": \"PT2S\",\n" + + " \"retryPeriod\": \"PT1S\"\n" + + "}\n", + new K8sDiscoveryConfig( + "test-cluster", + "PODNAMETEST", + "PODNAMESPACETEST", + "coordinatorns", + "overlordns", + Duration.millis(3000), + Duration.millis(2000), + Duration.millis(1000) + ) + ); + } + + private void testSerde(String jsonStr, K8sDiscoveryConfig expected) throws Exception + { + K8sDiscoveryConfig actual = jsonMapper.readValue( + jsonMapper.writeValueAsString( + jsonMapper.readValue(jsonStr, K8sDiscoveryConfig.class) + ), + K8sDiscoveryConfig.class + ); + + Assert.assertEquals(expected, actual); + } +} diff --git a/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidLeaderElectionIntTest.java b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidLeaderElectionIntTest.java new file mode 100644 index 000000000000..168c0625cda3 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidLeaderElectionIntTest.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.util.Config; +import org.apache.druid.discovery.DiscoveryDruidNode; +import org.apache.druid.discovery.DruidLeaderSelector; +import org.apache.druid.discovery.NodeRole; +import org.apache.druid.java.util.emitter.EmittingLogger; +import org.apache.druid.server.DruidNode; +import org.joda.time.Duration; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * This is not a UT, but very helpful when making changes to ensure things work with real K8S Api Server. + * It is ignored in the build but checked in the reporitory for running manually by devs. + */ +@Ignore("Needs K8S API Server") +public class K8sDruidLeaderElectionIntTest +{ + private final DiscoveryDruidNode testNode1 = new DiscoveryDruidNode( + new DruidNode("druid/router", "test-host1", true, 80, null, true, false), + NodeRole.ROUTER, + null + ); + + private final DiscoveryDruidNode testNode2 = new DiscoveryDruidNode( + new DruidNode("druid/router", "test-host2", true, 80, null, true, false), + NodeRole.ROUTER, + null + ); + + private final K8sDiscoveryConfig discoveryConfig = new K8sDiscoveryConfig("druid-cluster", null, null, "default", "default", + Duration.millis(10_000), Duration.millis(7_000), Duration.millis(3_000)); + + private final ApiClient k8sApiClient; + + private final String lockResourceName = "druid-leader-election"; + + public K8sDruidLeaderElectionIntTest() throws Exception + { + EmittingLogger.registerEmitter(new NoopServiceEmitter()); + k8sApiClient = Config.defaultClient(); + } + + // Note: This one is supposed to crash. + @Test(timeout = 60000L) + public void test_becomeLeader_exception() throws Exception + { + K8sDruidLeaderSelector leaderSelector = new K8sDruidLeaderSelector(testNode1.getDruidNode(), lockResourceName, discoveryConfig.getCoordinatorLeaderElectionConfigMapNamespace(), discoveryConfig, new DefaultK8sLeaderElectorFactory(k8sApiClient, discoveryConfig)); + + CountDownLatch becomeLeaderLatch = new CountDownLatch(1); + CountDownLatch stopBeingLeaderLatch = new CountDownLatch(1); + + AtomicBoolean failed = new AtomicBoolean(false); + + leaderSelector.registerListener(new DruidLeaderSelector.Listener() + { + @Override + public void becomeLeader() + { + becomeLeaderLatch.countDown(); + // This leads to a System.exit() and pod restart is expected to happen. + throw new RuntimeException("Leader crashed"); + } + + @Override + public void stopBeingLeader() + { + try { + becomeLeaderLatch.await(); + stopBeingLeaderLatch.countDown(); + } + catch (InterruptedException ex) { + failed.set(true); + } + } + }); + + becomeLeaderLatch.await(); + stopBeingLeaderLatch.await(); + Assert.assertFalse(failed.get()); + } + + @Test(timeout = 60000L) + public void test_leaderCandidate_stopped() throws Exception + { + K8sDruidLeaderSelector leaderSelector = new K8sDruidLeaderSelector(testNode1.getDruidNode(), lockResourceName, discoveryConfig.getCoordinatorLeaderElectionConfigMapNamespace(), discoveryConfig, new DefaultK8sLeaderElectorFactory(k8sApiClient, discoveryConfig)); + + CountDownLatch becomeLeaderLatch = new CountDownLatch(1); + CountDownLatch stopBeingLeaderLatch = new CountDownLatch(1); + + AtomicBoolean failed = new AtomicBoolean(false); + + leaderSelector.registerListener(new DruidLeaderSelector.Listener() + { + @Override + public void becomeLeader() + { + becomeLeaderLatch.countDown(); + } + + @Override + public void stopBeingLeader() + { + try { + becomeLeaderLatch.await(); + stopBeingLeaderLatch.countDown(); + } + catch (InterruptedException ex) { + failed.set(true); + } + } + }); + + becomeLeaderLatch.await(); + + leaderSelector.unregisterListener(); + + stopBeingLeaderLatch.await(); + Assert.assertFalse(failed.get()); + + leaderSelector = new K8sDruidLeaderSelector(testNode2.getDruidNode(), lockResourceName, discoveryConfig.getCoordinatorLeaderElectionConfigMapNamespace(), discoveryConfig, new DefaultK8sLeaderElectorFactory(k8sApiClient, discoveryConfig)); + + CountDownLatch becomeLeaderLatch2 = new CountDownLatch(1); + + leaderSelector.registerListener(new DruidLeaderSelector.Listener() + { + @Override + public void becomeLeader() + { + becomeLeaderLatch2.countDown(); + } + + @Override + public void stopBeingLeader() + { + } + }); + + becomeLeaderLatch2.await(); + } +} diff --git a/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidLeaderSelectorTest.java b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidLeaderSelectorTest.java new file mode 100644 index 000000000000..a500502524fb --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidLeaderSelectorTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import org.apache.druid.discovery.DiscoveryDruidNode; +import org.apache.druid.discovery.DruidLeaderSelector; +import org.apache.druid.discovery.NodeRole; +import org.apache.druid.server.DruidNode; +import org.joda.time.Duration; +import org.junit.Assert; +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; + +public class K8sDruidLeaderSelectorTest +{ + private final DiscoveryDruidNode testNode1 = new DiscoveryDruidNode( + new DruidNode("druid/router", "test-host1", true, 80, null, true, false), + NodeRole.ROUTER, + null + ); + + private final K8sDiscoveryConfig discoveryConfig = new K8sDiscoveryConfig("druid-cluster", null, null, + "default", "default", Duration.millis(10_000), Duration.millis(7_000), Duration.millis(3_000)); + + private final String lockResourceName = "druid-leader-election"; + + @Test(timeout = 5_000) + public void testLeaderElection_HappyPath() throws Exception + { + K8sDruidLeaderSelector leaderSelector = new K8sDruidLeaderSelector( + testNode1.getDruidNode(), + lockResourceName, + discoveryConfig.getCoordinatorLeaderElectionConfigMapNamespace(), + discoveryConfig, + new K8sLeaderElectorFactory() + { + @Override + public K8sLeaderElector create(String candidateId, String namespace, String lockResourceName) + { + return new K8sLeaderElector() + { + @Override + public String getCurrentLeader() + { + return testNode1.getDruidNode().getHostAndPortToUse(); + } + + @Override + public void run(Runnable startLeadingHook, Runnable stopLeadingHook) + { + startLeadingHook.run(); + try { + Thread.sleep(Long.MAX_VALUE); + } + catch (InterruptedException ex) { + stopLeadingHook.run(); + } + } + }; + } + } + ); + + Assert.assertEquals(testNode1.getDruidNode().getHostAndPortToUse(), leaderSelector.getCurrentLeader()); + + CountDownLatch becomeLeaderLatch = new CountDownLatch(1); + CountDownLatch stopBeingLeaderLatch = new CountDownLatch(1); + + leaderSelector.registerListener( + new DruidLeaderSelector.Listener() + { + @Override + public void becomeLeader() + { + becomeLeaderLatch.countDown(); + } + + @Override + public void stopBeingLeader() + { + stopBeingLeaderLatch.countDown(); + } + } + ); + + becomeLeaderLatch.await(); + leaderSelector.unregisterListener(); + stopBeingLeaderLatch.await(); + } + + @Test(timeout = 5_000) + public void testLeaderElection_LeaderElectorExits() throws Exception + { + K8sDruidLeaderSelector leaderSelector = new K8sDruidLeaderSelector( + testNode1.getDruidNode(), + lockResourceName, + discoveryConfig.getCoordinatorLeaderElectionConfigMapNamespace(), + discoveryConfig, + new K8sLeaderElectorFactory() + { + @Override + public K8sLeaderElector create(String candidateId, String namespace, String lockResourceName) + { + return new K8sLeaderElector() + { + private boolean isFirstTime = true; + + @Override + public String getCurrentLeader() + { + return testNode1.getDruidNode().getHostAndPortToUse(); + } + + @Override + public void run(Runnable startLeadingHook, Runnable stopLeadingHook) + { + startLeadingHook.run(); + + if (isFirstTime) { + isFirstTime = false; + stopLeadingHook.run(); + } else { + try { + Thread.sleep(Long.MAX_VALUE); + } + catch (InterruptedException ex) { + stopLeadingHook.run(); + } + } + } + }; + } + } + ); + + Assert.assertEquals(testNode1.getDruidNode().getHostAndPortToUse(), leaderSelector.getCurrentLeader()); + + CountDownLatch becomeLeaderLatch = new CountDownLatch(2); + CountDownLatch stopBeingLeaderLatch = new CountDownLatch(2); + + leaderSelector.registerListener( + new DruidLeaderSelector.Listener() + { + @Override + public void becomeLeader() + { + becomeLeaderLatch.countDown(); + } + + @Override + public void stopBeingLeader() + { + stopBeingLeaderLatch.countDown(); + } + } + ); + + becomeLeaderLatch.await(); + leaderSelector.unregisterListener(); + stopBeingLeaderLatch.await(); + } +} diff --git a/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncerTest.java b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncerTest.java new file mode 100644 index 000000000000..445458be7f5c --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidNodeAnnouncerTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import org.apache.druid.discovery.DiscoveryDruidNode; +import org.apache.druid.discovery.NodeRole; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.server.DruidNode; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +public class K8sDruidNodeAnnouncerTest +{ + private final DiscoveryDruidNode testNode = new DiscoveryDruidNode( + new DruidNode("druid/router", "test-host", true, 80, null, true, false), + NodeRole.ROUTER, + null + ); + + private final ObjectMapper jsonMapper = new DefaultObjectMapper(); + + private final PodInfo podInfo = new PodInfo("testpod", "testns"); + + private final K8sDiscoveryConfig discoveryConfig = new K8sDiscoveryConfig("druid-cluster", null, null, null, null, null, null, null); + + @Test + public void testAnnounce() throws Exception + { + K8sApiClient mockK8sApiClient = EasyMock.createMock(K8sApiClient.class); + Capture podNameArg = Capture.newInstance(); + Capture namespaceArg = Capture.newInstance(); + Capture patchArg = Capture.newInstance(); + mockK8sApiClient.patchPod(EasyMock.capture(podNameArg), EasyMock.capture(namespaceArg), EasyMock.capture(patchArg)); + EasyMock.replay(mockK8sApiClient); + + K8sDruidNodeAnnouncer announcer = new K8sDruidNodeAnnouncer(podInfo, discoveryConfig, mockK8sApiClient, jsonMapper); + announcer.announce(testNode); + + Assert.assertEquals(podInfo.getPodName(), podNameArg.getValue()); + Assert.assertEquals(podInfo.getPodNamespace(), namespaceArg.getValue()); + + List> actualPatchList = jsonMapper.readValue( + patchArg.getValue(), + new TypeReference>>() + { + } + ); + + List> expectedPatchList = Lists.newArrayList( + ImmutableMap.of( + "op", "add", + "path", "/metadata/labels/druidDiscoveryAnnouncement-router", + "value", "true" + ), + ImmutableMap.of( + "op", "add", + "path", "/metadata/labels/druidDiscoveryAnnouncement-id", + "value", "test-host-80" + ), + ImmutableMap.of( + "op", "add", + "path", "/metadata/labels/druidDiscoveryAnnouncement-cluster-identifier", + "value", discoveryConfig.getClusterIdentifier() + ), + ImmutableMap.of( + "op", "add", + "path", "/metadata/annotations/druidNodeInfo-router", + "value", jsonMapper.writeValueAsString(testNode) + ) + ); + Assert.assertEquals(expectedPatchList, actualPatchList); + } + + @Test + public void testUnannounce() throws Exception + { + K8sApiClient mockK8sApiClient = EasyMock.createMock(K8sApiClient.class); + Capture podNameArg = Capture.newInstance(); + Capture namespaceArg = Capture.newInstance(); + Capture patchArg = Capture.newInstance(); + mockK8sApiClient.patchPod(EasyMock.capture(podNameArg), EasyMock.capture(namespaceArg), EasyMock.capture(patchArg)); + EasyMock.replay(mockK8sApiClient); + + K8sDruidNodeAnnouncer announcer = new K8sDruidNodeAnnouncer(podInfo, discoveryConfig, mockK8sApiClient, jsonMapper); + announcer.unannounce(testNode); + + Assert.assertEquals(podInfo.getPodName(), podNameArg.getValue()); + Assert.assertEquals(podInfo.getPodNamespace(), namespaceArg.getValue()); + + List> actualPatchList = jsonMapper.readValue( + patchArg.getValue(), + new TypeReference>>() + { + } + ); + + List> expectedPatchList = Lists.newArrayList( + ImmutableMap.of( + "op", "remove", + "path", "/metadata/labels/druidDiscoveryAnnouncement-router" + ), + ImmutableMap.of( + "op", "remove", + "path", "/metadata/labels/druidDiscoveryAnnouncement-id" + ), + ImmutableMap.of( + "op", "remove", + "path", "/metadata/labels/druidDiscoveryAnnouncement-cluster-identifier" + ), + ImmutableMap.of( + "op", "remove", + "path", "/metadata/annotations/druidNodeInfo-router" + ) + ); + Assert.assertEquals(expectedPatchList, actualPatchList); + } +} diff --git a/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidNodeDiscoveryProviderTest.java b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidNodeDiscoveryProviderTest.java new file mode 100644 index 000000000000..1a9cfbc9e58b --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/K8sDruidNodeDiscoveryProviderTest.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import io.kubernetes.client.util.Watch; +import org.apache.druid.discovery.DiscoveryDruidNode; +import org.apache.druid.discovery.DruidNodeDiscovery; +import org.apache.druid.discovery.NodeRole; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.server.DruidNode; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Test; + +import java.net.SocketTimeoutException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class K8sDruidNodeDiscoveryProviderTest +{ + private static final Logger LOGGER = new Logger(K8sDruidNodeDiscoveryProviderTest.class); + + private final DiscoveryDruidNode testNode1 = new DiscoveryDruidNode( + new DruidNode("druid/router", "test-host1", true, 80, null, true, false), + NodeRole.ROUTER, + null + ); + + private final DiscoveryDruidNode testNode2 = new DiscoveryDruidNode( + new DruidNode("druid/router", "test-host2", true, 80, null, true, false), + NodeRole.ROUTER, + null + ); + + private final DiscoveryDruidNode testNode3 = new DiscoveryDruidNode( + new DruidNode("druid/router", "test-host3", true, 80, null, true, false), + NodeRole.ROUTER, + null + ); + + private final DiscoveryDruidNode testNode4 = new DiscoveryDruidNode( + new DruidNode("druid/router", "test-host4", true, 80, null, true, false), + NodeRole.ROUTER, + null + ); + + private final DiscoveryDruidNode testNode5 = new DiscoveryDruidNode( + new DruidNode("druid/router", "test-host5", true, 80, null, true, false), + NodeRole.ROUTER, + null + ); + + private final PodInfo podInfo = new PodInfo("testpod", "testns"); + + private final K8sDiscoveryConfig discoveryConfig = new K8sDiscoveryConfig("druid-cluster", null, null, null, null, null, null, null); + + @Test(timeout = 60_000) + public void testGetForNodeRole() throws Exception + { + String labelSelector = "druidDiscoveryAnnouncement-cluster-identifier=druid-cluster,druidDiscoveryAnnouncement-router=true"; + K8sApiClient mockK8sApiClient = EasyMock.createMock(K8sApiClient.class); + EasyMock.expect(mockK8sApiClient.listPods(podInfo.getPodNamespace(), labelSelector, NodeRole.ROUTER)).andReturn( + new DiscoveryDruidNodeList( + "v1", + ImmutableMap.of( + testNode1.getDruidNode().getHostAndPortToUse(), testNode1, + testNode2.getDruidNode().getHostAndPortToUse(), testNode2 + ) + ) + ); + EasyMock.expect(mockK8sApiClient.watchPods( + podInfo.getPodNamespace(), labelSelector, "v1", NodeRole.ROUTER)).andReturn(null); + EasyMock.expect(mockK8sApiClient.listPods(podInfo.getPodNamespace(), labelSelector, NodeRole.ROUTER)).andReturn( + new DiscoveryDruidNodeList( + "v2", + ImmutableMap.of( + testNode2.getDruidNode().getHostAndPortToUse(), testNode2, + testNode3.getDruidNode().getHostAndPortToUse(), testNode3 + ) + ) + ); + EasyMock.expect(mockK8sApiClient.watchPods( + podInfo.getPodNamespace(), labelSelector, "v2", NodeRole.ROUTER)).andReturn( + new MockWatchResult(Collections.emptyList(), true, false) + ); + EasyMock.expect(mockK8sApiClient.watchPods( + podInfo.getPodNamespace(), labelSelector, "v2", NodeRole.ROUTER)).andReturn( + new MockWatchResult( + ImmutableList.of( + new Watch.Response<>(WatchResult.ADDED, new DiscoveryDruidNodeAndResourceVersion("v3", testNode4)), + new Watch.Response<>(WatchResult.DELETED, new DiscoveryDruidNodeAndResourceVersion("v4", testNode2)) + ), + false, + true + ) + ); + EasyMock.expect(mockK8sApiClient.watchPods( + podInfo.getPodNamespace(), labelSelector, "v4", NodeRole.ROUTER)).andReturn( + new MockWatchResult( + ImmutableList.of( + new Watch.Response<>(WatchResult.ADDED, new DiscoveryDruidNodeAndResourceVersion("v5", testNode5)), + new Watch.Response<>(WatchResult.DELETED, new DiscoveryDruidNodeAndResourceVersion("v6", testNode3)) + ), + false, + false + ) + ); + EasyMock.replay(mockK8sApiClient); + + K8sDruidNodeDiscoveryProvider discoveryProvider = new K8sDruidNodeDiscoveryProvider( + podInfo, + discoveryConfig, + mockK8sApiClient, + 1 + ); + discoveryProvider.start(); + + K8sDruidNodeDiscoveryProvider.NodeRoleWatcher nodeDiscovery = discoveryProvider.getForNodeRole(NodeRole.ROUTER, false); + + MockListener testListener = new MockListener( + ImmutableList.of( + MockListener.Event.added(testNode1), + MockListener.Event.added(testNode2), + MockListener.Event.inited(), + MockListener.Event.added(testNode3), + MockListener.Event.deleted(testNode1), + MockListener.Event.added(testNode4), + MockListener.Event.deleted(testNode2), + MockListener.Event.added(testNode5), + MockListener.Event.deleted(testNode3) + ) + ); + nodeDiscovery.registerListener(testListener); + + nodeDiscovery.start(); + + testListener.assertSuccess(); + + discoveryProvider.stop(); + } + + private static class MockListener implements DruidNodeDiscovery.Listener + { + List events; + private boolean failed = false; + private String failReason; + + public MockListener(List events) + { + this.events = Lists.newArrayList(events); + } + + @Override + public void nodeViewInitialized() + { + assertNextEvent(Event.inited()); + } + + @Override + public void nodesAdded(Collection nodes) + { + List l = Lists.newArrayList(nodes); + Collections.sort(l, (n1, n2) -> n1.getDruidNode().getHostAndPortToUse().compareTo(n2.getDruidNode().getHostAndPortToUse())); + + for (DiscoveryDruidNode node : l) { + assertNextEvent(Event.added(node)); + } + } + + @Override + public void nodesRemoved(Collection nodes) + { + List l = Lists.newArrayList(nodes); + Collections.sort(l, (n1, n2) -> n1.getDruidNode().getHostAndPortToUse().compareTo(n2.getDruidNode().getHostAndPortToUse())); + + for (DiscoveryDruidNode node : l) { + assertNextEvent(Event.deleted(node)); + } + } + + private void assertNextEvent(Event actual) + { + if (!failed && !events.isEmpty()) { + Event expected = events.remove(0); + failed = !actual.equals(expected); + if (failed) { + failReason = StringUtils.format("Failed Equals [%s] and [%s]", expected, actual); + } + } + } + + public void assertSuccess() throws Exception + { + while (!events.isEmpty()) { + Assert.assertFalse(failReason, failed); + LOGGER.info("Waiting for events to finish."); + Thread.sleep(1000); + } + + Assert.assertFalse(failReason, failed); + } + + static class Event + { + String type; + DiscoveryDruidNode node; + + private Event(String type, DiscoveryDruidNode node) + { + this.type = type; + this.node = node; + } + + static Event inited() + { + return new Event("inited", null); + } + + static Event added(DiscoveryDruidNode node) + { + return new Event("added", node); + } + + static Event deleted(DiscoveryDruidNode node) + { + return new Event("deleted", node); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Event event = (Event) o; + return type.equals(event.type) && + Objects.equals(node, event.node); + } + + @Override + public int hashCode() + { + return Objects.hash(type, node); + } + + @Override + public String toString() + { + return "Event{" + + "type='" + type + '\'' + + ", node=" + node + + '}'; + } + } + } + + private static class MockWatchResult implements WatchResult + { + private List> results; + + private volatile boolean timeoutOnStart; + private volatile boolean timeooutOnEmptyResults; + private volatile boolean closeCalled = false; + + public MockWatchResult( + List> results, + boolean timeoutOnStart, + boolean timeooutOnEmptyResults + ) + { + this.results = Lists.newArrayList(results); + this.timeoutOnStart = timeoutOnStart; + this.timeooutOnEmptyResults = timeooutOnEmptyResults; + } + + @Override + public boolean hasNext() throws SocketTimeoutException + { + if (timeoutOnStart) { + throw new SocketTimeoutException("testing timeout on start!!!"); + } + + if (results.isEmpty()) { + if (timeooutOnEmptyResults) { + throw new SocketTimeoutException("testing timeout on end!!!"); + } else { + try { + Thread.sleep(Long.MAX_VALUE); + return false; // just making compiler happy, will never reach this. + } + catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + } + } else { + return true; + } + } + + @Override + public Watch.Response next() + { + return results.remove(0); + } + + @Override + public void close() + { + closeCalled = true; + } + + public void assertSuccess() + { + Assert.assertTrue("close() not called", closeCalled); + } + } +} diff --git a/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/NoopServiceEmitter.java b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/NoopServiceEmitter.java new file mode 100644 index 000000000000..f5adcd90e831 --- /dev/null +++ b/extensions-core/kubernetes-extensions/src/test/java/org/apache/druid/k8s/discovery/NoopServiceEmitter.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.k8s.discovery; + +import org.apache.druid.java.util.emitter.core.Event; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; + +public class NoopServiceEmitter extends ServiceEmitter +{ + public NoopServiceEmitter() + { + super("", "", null); + } + + @Override + public void emit(Event event) + { + } +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java new file mode 100644 index 000000000000..1367163d602b --- /dev/null +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueTest.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.druid.indexer.TaskStatus; +import org.apache.druid.indexing.common.TaskToolbox; +import org.apache.druid.indexing.common.actions.TaskActionClient; +import org.apache.druid.indexing.common.actions.TaskActionClientFactory; +import org.apache.druid.indexing.common.task.AbstractBatchIndexTask; +import org.apache.druid.indexing.common.task.IngestionTestBase; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.autoscaling.ScalingStats; +import org.apache.druid.indexing.overlord.config.TaskLockConfig; +import org.apache.druid.indexing.overlord.config.TaskQueueConfig; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.apache.druid.timeline.DataSegment; +import org.joda.time.Interval; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.annotation.Nullable; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Executor; + +public class TaskQueueTest extends IngestionTestBase +{ + private static final Granularity SEGMENT_GRANULARITY = Granularities.DAY; + + /** + * This test verifies releasing all locks of a task when it is not ready to run yet. + * + * This test uses 2 APIs, {@link TaskQueue} APIs and {@link IngestionTestBase} APIs + * to emulate the scenario of deadlock. The IngestionTestBase provides low-leve APIs + * which you can manipulate {@link TaskLockbox} manually. These APIs should be used + * only to emulate a certain deadlock scenario. All normal tasks should use TaskQueue + * APIs. + */ + @Test + public void testManageInternalReleaseLockWhenTaskIsNotReady() throws Exception + { + final TaskActionClientFactory actionClientFactory = createActionClientFactory(); + final TaskQueue taskQueue = new TaskQueue( + new TaskLockConfig(), + new TaskQueueConfig(null, null, null, null), + getTaskStorage(), + new SimpleTaskRunner(actionClientFactory), + actionClientFactory, + getLockbox(), + new NoopServiceEmitter() + ); + taskQueue.setActive(true); + // task1 emulates a case when there is a task that was issued before task2 and acquired locks conflicting + // to task2. + final TestTask task1 = new TestTask("t1", Intervals.of("2021-01/P1M")); + // Manually get locks for task1. task2 cannot be ready because of task1. + prepareTaskForLocking(task1); + Assert.assertTrue(task1.isReady(actionClientFactory.create(task1))); + + final TestTask task2 = new TestTask("t2", Intervals.of("2021-01-31/P1M")); + taskQueue.add(task2); + taskQueue.manageInternal(); + Assert.assertFalse(task2.isDone()); + Assert.assertTrue(getLockbox().findLocksForTask(task2).isEmpty()); + + // task3 can run because task2 is still blocked by task1. + final TestTask task3 = new TestTask("t3", Intervals.of("2021-02-01/P1M")); + taskQueue.add(task3); + taskQueue.manageInternal(); + Assert.assertFalse(task2.isDone()); + Assert.assertTrue(task3.isDone()); + Assert.assertTrue(getLockbox().findLocksForTask(task2).isEmpty()); + + // Shut down task1 and task3 and release their locks. + shutdownTask(task1); + taskQueue.shutdown(task3.getId(), "Emulating shutdown of task3"); + + // Now task2 should run. + taskQueue.manageInternal(); + Assert.assertTrue(task2.isDone()); + } + + private static class TestTask extends AbstractBatchIndexTask + { + private final Interval interval; + private boolean done; + + private TestTask(String id, Interval interval) + { + super(id, "datasource", null); + this.interval = interval; + } + + @Override + public boolean isReady(TaskActionClient taskActionClient) throws Exception + { + return tryTimeChunkLock(taskActionClient, ImmutableList.of(interval)); + } + + @Override + public TaskStatus runTask(TaskToolbox toolbox) + { + done = true; + return TaskStatus.success(getId()); + } + + @Override + public boolean requireLockExistingSegments() + { + return false; + } + + @Override + public List findSegmentsToLock(TaskActionClient taskActionClient, List intervals) + { + return null; + } + + @Override + public boolean isPerfectRollup() + { + return false; + } + + @Nullable + @Override + public Granularity getSegmentGranularity() + { + return SEGMENT_GRANULARITY; + } + + @Override + public String getType() + { + return "test"; + } + + public boolean isDone() + { + return done; + } + } + + private static class SimpleTaskRunner implements TaskRunner + { + private final TaskActionClientFactory actionClientFactory; + + private SimpleTaskRunner(TaskActionClientFactory actionClientFactory) + { + this.actionClientFactory = actionClientFactory; + } + + @Override + public List>> restore() + { + return null; + } + + @Override + public void start() + { + } + + @Override + public void registerListener(TaskRunnerListener listener, Executor executor) + { + } + + @Override + public void unregisterListener(String listenerId) + { + } + + @Override + public ListenableFuture run(Task task) + { + try { + final TaskToolbox toolbox = Mockito.mock(TaskToolbox.class); + Mockito.when(toolbox.getTaskActionClient()).thenReturn(actionClientFactory.create(task)); + return Futures.immediateFuture(task.run(toolbox)); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void shutdown(String taskid, String reason) + { + } + + @Override + public void stop() + { + } + + @Override + public Collection getRunningTasks() + { + return null; + } + + @Override + public Collection getPendingTasks() + { + return null; + } + + @Override + public Collection getKnownTasks() + { + return Collections.emptyList(); + } + + @Override + public Optional getScalingStats() + { + return null; + } + + @Override + public long getTotalTaskSlotCount() + { + return 0; + } + + @Override + public long getIdleTaskSlotCount() + { + return 0; + } + + @Override + public long getUsedTaskSlotCount() + { + return 0; + } + + @Override + public long getLazyTaskSlotCount() + { + return 0; + } + + @Override + public long getBlacklistedTaskSlotCount() + { + return 0; + } + } +} diff --git a/integration-tests/docker/docker-compose.cli-indexer.yml b/integration-tests/docker/docker-compose.cli-indexer.yml new file mode 100644 index 000000000000..0e4ba9c0ea26 --- /dev/null +++ b/integration-tests/docker/docker-compose.cli-indexer.yml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2.2" +services: + druid-zookeeper-kafka: + extends: + file: docker-compose.base.yml + service: druid-zookeeper-kafka + + druid-metadata-storage: + extends: + file: docker-compose.base.yml + service: druid-metadata-storage + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-zookeeper-kafka + + druid-overlord: + extends: + file: docker-compose.base.yml + service: druid-overlord + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-metadata-storage + - druid-zookeeper-kafka + + druid-coordinator: + extends: + file: docker-compose.base.yml + service: druid-coordinator + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-overlord + - druid-metadata-storage + - druid-zookeeper-kafka + + druid-historical: + extends: + file: docker-compose.base.yml + service: druid-historical + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-zookeeper-kafka + + druid-indexer: + extends: + file: docker-compose.base.yml + service: druid-indexer + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-zookeeper-kafka + - druid-overlord + + druid-broker: + extends: + file: docker-compose.base.yml + service: druid-broker + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-zookeeper-kafka + - druid-indexer + - druid-historical + + druid-router: + extends: + file: docker-compose.base.yml + service: druid-router + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-zookeeper-kafka + - druid-coordinator + - druid-broker + +networks: + druid-it-net: + name: druid-it-net + ipam: + config: + - subnet: 172.172.172.0/24 \ No newline at end of file diff --git a/integration-tests/docker/docker-compose.high-availability.yml b/integration-tests/docker/docker-compose.high-availability.yml new file mode 100644 index 000000000000..f9be36b7d03a --- /dev/null +++ b/integration-tests/docker/docker-compose.high-availability.yml @@ -0,0 +1,151 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "2.2" +services: + druid-zookeeper-kafka: + extends: + file: docker-compose.base.yml + service: druid-zookeeper-kafka + + druid-metadata-storage: + extends: + file: docker-compose.base.yml + service: druid-metadata-storage + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-zookeeper-kafka + + druid-coordinator: + extends: + file: docker-compose.base.yml + service: druid-coordinator + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + - DRUID_LOG_PATH=/shared/logs/ha-coordinator-one.log + - druid_manager_config_pollDuration=PT10S + - druid_manager_rules_pollDuration=PT10S + - druid_manager_segments_pollDuration=PT10S + - druid_coordinator_period=PT10S + depends_on: + - druid-metadata-storage + - druid-zookeeper-kafka + + druid-coordinator-two: + extends: + file: docker-compose.base.yml + service: druid-coordinator-two + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + - DRUID_LOG_PATH=/shared/logs/ha-coordinator-two.log + - druid_host=druid-coordinator-two + - druid_manager_config_pollDuration=PT10S + - druid_manager_rules_pollDuration=PT10S + - druid_manager_segments_pollDuration=PT10S + - druid_coordinator_period=PT10S + depends_on: + - druid-coordinator + - druid-metadata-storage + - druid-zookeeper-kafka + + druid-overlord: + extends: + file: docker-compose.base.yml + service: druid-overlord + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + - DRUID_LOG_PATH=/shared/logs/ha-overlord-one.log + depends_on: + - druid-coordinator + - druid-coordinator-two + - druid-metadata-storage + - druid-zookeeper-kafka + + druid-overlord-two: + extends: + file: docker-compose.base.yml + service: druid-overlord-two + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + - DRUID_LOG_PATH=/shared/logs/ha-overlord-two.log + - druid_host=druid-overlord-two + depends_on: + - druid-coordinator + - druid-coordinator-two + - druid-metadata-storage + - druid-zookeeper-kafka + + druid-broker: + extends: + file: docker-compose.base.yml + service: druid-broker + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-coordinator + - druid-coordinator-two + - druid-overlord + - druid-overlord-two + - druid-zookeeper-kafka + + druid-router: + extends: + file: docker-compose.base.yml + service: druid-router + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + depends_on: + - druid-coordinator + - druid-coordinator-two + - druid-overlord + - druid-overlord-two + - druid-broker + + druid-custom-node-role: + image: druid/cluster + container_name: druid-custom-node-role + networks: + druid-it-net: + ipv4_address: 172.172.172.90 + ports: + - 50011:50011 + - 9301:9301 + - 9501:9501 + privileged: true + volumes: + - ${HOME}/shared:/shared + - ./service-supervisords/druid.conf:/usr/lib/druid/conf/druid.conf + environment: + - DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP} + - DRUID_SERVICE=custom-node-role + - DRUID_LOG_PATH=/shared/logs/custom-node-role.log + - SERVICE_DRUID_JAVA_OPTS=-server -Xmx32m -Xms32m -XX:+UseG1GC -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5011 + - druid_host=druid-custom-node-role + - druid_auth_basic_common_cacheDirectory=/tmp/authCache/custom_node_role + - druid_server_https_crlPath=/tls/revocations.crl + env_file: + - ./environment-configs/common + depends_on: + - druid-zookeeper-kafka + - druid-coordinator + - druid-coordinator-two + +networks: + druid-it-net: + name: druid-it-net + ipam: + config: + - subnet: 172.172.172.0/24 \ No newline at end of file diff --git a/integration-tests/docker/environment-configs/indexer b/integration-tests/docker/environment-configs/indexer new file mode 100644 index 000000000000..906fe70cc510 --- /dev/null +++ b/integration-tests/docker/environment-configs/indexer @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +DRUID_SERVICE=indexer +DRUID_LOG_PATH=/shared/logs/indexer.log + +# JAVA OPTS +SERVICE_DRUID_JAVA_OPTS=-server -Xmx512m -Xms512m -XX:+UseG1GC -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5008 + +# Druid configs +druid_host=druid-indexer +druid_server_http_numThreads=4 +druid_storage_storageDirectory=/shared/storage + +druid_processing_buffer_sizeBytes=25000000 +druid_processing_numThreads=1 +druid_selectors_indexing_serviceName=druid/overlord +druid_indexer_task_chathandler_type=announce +druid_auth_basic_common_cacheDirectory=/tmp/authCache/indexer +druid_startup_logging_logProperties=true +druid_server_https_crlPath=/tls/revocations.crl +druid_worker_capacity=10 diff --git a/integration-tests/docker/test-data/high-availability-sample-data.sql b/integration-tests/docker/test-data/high-availability-sample-data.sql new file mode 100644 index 000000000000..18ab48ad556b --- /dev/null +++ b/integration-tests/docker/test-data/high-availability-sample-data.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload) VALUES ('twitterstream_2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z_2013-01-02T04:13:41.980Z_v9','twitterstream','2013-05-13T01:08:18.192Z','2013-01-01T00:00:00.000Z','2013-01-02T00:00:00.000Z',0,'2013-01-02T04:13:41.980Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-01T00:00:00.000Z/2013-01-02T00:00:00.000Z\",\"version\":\"2013-01-02T04:13:41.980Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z/2013-01-02T04:13:41.980Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":445235220,\"identifier\":\"twitterstream_2013-01-01T00:00:00.000Z_2013-01-02T00:00:00.000Z_2013-01-02T04:13:41.980Z_v9\"}'); +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload) VALUES ('twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9','twitterstream','2013-05-13T00:03:28.640Z','2013-01-02T00:00:00.000Z','2013-01-03T00:00:00.000Z',0,'2013-01-03T03:44:58.791Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-02T00:00:00.000Z/2013-01-03T00:00:00.000Z\",\"version\":\"2013-01-03T03:44:58.791Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z/2013-01-03T03:44:58.791Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":435325540,\"identifier\":\"twitterstream_2013-01-02T00:00:00.000Z_2013-01-03T00:00:00.000Z_2013-01-03T03:44:58.791Z_v9\"}'); +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload) VALUES ('twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9','twitterstream','2013-05-13T00:03:48.807Z','2013-01-03T00:00:00.000Z','2013-01-04T00:00:00.000Z',0,'2013-01-04T04:09:13.590Z_v9',1,'{\"dataSource\":\"twitterstream\",\"interval\":\"2013-01-03T00:00:00.000Z/2013-01-04T00:00:00.000Z\",\"version\":\"2013-01-04T04:09:13.590Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/twitterstream/2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z/2013-01-04T04:09:13.590Z_v9/0/index.zip\"},\"dimensions\":\"has_links,first_hashtag,user_time_zone,user_location,has_mention,user_lang,rt_name,user_name,is_retweet,is_viral,has_geo,url_domain,user_mention_name,reply_to_name\",\"metrics\":\"count,tweet_length,num_followers,num_links,num_mentions,num_hashtags,num_favorites,user_total_tweets\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":411651320,\"identifier\":\"twitterstream_2013-01-03T00:00:00.000Z_2013-01-04T00:00:00.000Z_2013-01-04T04:09:13.590Z_v9\"}'); +INSERT INTO druid_segments (id,dataSource,created_date,start,end,partitioned,version,used,payload) VALUES ('wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9','wikipedia_editstream','2013-03-15T20:49:52.348Z','2012-12-29T00:00:00.000Z','2013-01-10T08:00:00.000Z',0,'2013-01-10T08:13:47.830Z_v9',1,'{\"dataSource\":\"wikipedia_editstream\",\"interval\":\"2012-12-29T00:00:00.000Z/2013-01-10T08:00:00.000Z\",\"version\":\"2013-01-10T08:13:47.830Z_v9\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia_editstream/2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z/2013-01-10T08:13:47.830Z_v9/0/index.zip\"},\"dimensions\":\"anonymous,area_code,city,continent_code,country_name,dma_code,geo,language,namespace,network,newpage,page,postal_code,region_lookup,robot,unpatrolled,user\",\"metrics\":\"added,count,deleted,delta,delta_hist,unique_users,variation\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":446027801,\"identifier\":\"wikipedia_editstream_2012-12-29T00:00:00.000Z_2013-01-10T08:00:00.000Z_2013-01-10T08:13:47.830Z_v9\"}'); +INSERT INTO druid_segments (id, dataSource, created_date, start, end, partitioned, version, used, payload) VALUES ('wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z', 'wikipedia', '2013-08-08T21:26:23.799Z', '2013-08-01T00:00:00.000Z', '2013-08-02T00:00:00.000Z', '0', '2013-08-08T21:22:48.989Z', '1', '{\"dataSource\":\"wikipedia\",\"interval\":\"2013-08-01T00:00:00.000Z/2013-08-02T00:00:00.000Z\",\"version\":\"2013-08-08T21:22:48.989Z\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia/20130801T000000.000Z_20130802T000000.000Z/2013-08-08T21_22_48.989Z/0/index.zip\"},\"dimensions\":\"dma_code,continent_code,geo,area_code,robot,country_name,network,city,namespace,anonymous,unpatrolled,page,postal_code,language,newpage,user,region_lookup\",\"metrics\":\"count,delta,variation,added,deleted\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":24664730,\"identifier\":\"wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z\"}'); diff --git a/integration-tests/k8s/role-and-binding.yaml b/integration-tests/k8s/role-and-binding.yaml new file mode 100644 index 000000000000..ef15b6b74ee1 --- /dev/null +++ b/integration-tests/k8s/role-and-binding.yaml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: druid-cluster +rules: +- apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - '*' +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: druid-cluster +subjects: +- kind: ServiceAccount + name: default +roleRef: + kind: Role + name: druid-cluster + apiGroup: rbac.authorization.k8s.io diff --git a/integration-tests/k8s/tiny-cluster.yaml b/integration-tests/k8s/tiny-cluster.yaml new file mode 100644 index 000000000000..98dc6231fac9 --- /dev/null +++ b/integration-tests/k8s/tiny-cluster.yaml @@ -0,0 +1,347 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: "druid.apache.org/v1alpha1" +kind: "Druid" +metadata: + name: tiny-cluster +spec: + image: druid/cluster:v1 + # Optionally specify image for all nodes. Can be specify on nodes also + # imagePullSecrets: + # - name: tutu + startScript: /druid.sh + podLabels: + environment: stage + release: alpha + podAnnotations: + dummy: k8s_extn_needs_atleast_one_annotation + readinessProbe: + httpGet: + path: /status/health + port: 8088 + securityContext: + fsGroup: 0 + runAsUser: 0 + runAsGroup: 0 + containerSecurityContext: + privileged: true + services: + - spec: + type: ClusterIP + clusterIP: None + commonConfigMountPath: "/opt/druid/conf/druid/cluster/_common" + jvm.options: |- + -server + -XX:MaxDirectMemorySize=10240g + -Duser.timezone=UTC + -Dfile.encoding=UTF-8 + -Dlog4j.debug + -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager + log4j.config: |- + + + + + + + + + + + + + + common.runtime.properties: | + + # + # Zookeeper-less Druid Cluster + # + druid.zk.service.enabled=false + druid.discovery.type=k8s + druid.discovery.k8s.clusterIdentifier=druid-it + druid.serverview.type=http + druid.coordinator.loadqueuepeon.type=http + druid.indexer.runner.type=httpRemote + + # Metadata Store + druid.metadata.storage.type=derby + druid.metadata.storage.connector.connectURI=jdbc:derby://localhost:1527/var/druid/metadata.db;create=true + druid.metadata.storage.connector.host=localhost + druid.metadata.storage.connector.port=1527 + druid.metadata.storage.connector.createTables=true + + # Deep Storage + druid.storage.type=local + druid.storage.storageDirectory=/druid/data/deepstorage + + # + # Extensions + # + druid.extensions.loadList=["druid-avro-extensions","druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-kubernetes-extensions"] + + # + # Service discovery + # + druid.selectors.indexing.serviceName=druid/overlord + druid.selectors.coordinator.serviceName=druid/coordinator + + druid.indexer.logs.type=file + druid.indexer.logs.directory=/druid/data/task-logs + druid.indexer.task.baseDir=/druid/data/task-base + + druid.lookup.enableLookupSyncOnStartup=false + + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + + nodes: + brokers: + # Optionally specify for running broker as Deployment + # kind: Deployment + nodeType: "broker" + # Optionally specify for broker nodes + # imagePullSecrets: + # - name: tutu + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: broker-service-port + nodePort: 30100 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-brokers + metadata: + name: broker-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/broker" + replicas: 1 + runtime.properties: | + druid.service=druid/broker + + # HTTP server threads + druid.broker.http.numConnections=5 + druid.server.http.numThreads=40 + + # Processing threads and buffers + druid.processing.buffer.sizeBytes=25000000 + druid.processing.numThreads=1 + druid.sql.enable=true + extra.jvm.options: |- + -Xmx512m + -Xms512m + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: REPLACE_VOLUMES/tmp + resources: + requests: + memory: "800Mi" + limits: + memory: "800Mi" + + coordinators: + # Optionally specify for running coordinator as Deployment + # kind: Deployment + nodeType: "coordinator" + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: coordinator-service-port + nodePort: 30200 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-coordinators + metadata: + name: coordinator-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/master/coordinator-overlord" + replicas: 1 + runtime.properties: | + druid.service=druid/coordinator + + # HTTP server threads + druid.coordinator.startDelay=PT30S + druid.coordinator.period=PT30S + + # Configure this coordinator to also run as Overlord + druid.coordinator.asOverlord.enabled=true + druid.coordinator.asOverlord.overlordService=druid/overlord + druid.indexer.queue.startDelay=PT30S + extra.jvm.options: |- + -Xmx800m + -Xms800m + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: REPLACE_VOLUMES/tmp + resources: + requests: + memory: "1G" + limits: + memory: "1G" + + historicals: + nodeType: "historical" + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: historical-service-port + nodePort: 30300 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-historicals + metadata: + name: historical-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/historical" + replicas: 1 + runtime.properties: | + druid.service=druid/historical + druid.processing.buffer.sizeBytes=25000000 + druid.processing.numThreads=2 + # Segment storage + druid.segmentCache.locations=[{"path":"/druid/data/segments","maxSize":10737418240}] + druid.server.maxSize=10737418240 + extra.jvm.options: |- + -Xmx512m + -Xms512m + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: REPLACE_VOLUMES/tmp + resources: + requests: + memory: "1G" + limits: + memory: "1G" + + routers: + nodeType: "router" + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: router-service-port + nodePort: 30400 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-routers + metadata: + name: router-%s-service + - spec: + type: ClusterIP + clusterIP: None + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/router" + replicas: 1 + runtime.properties: | + druid.service=druid/router + druid.plaintextPort=8088 + + # HTTP proxy + druid.router.http.numConnections=50 + druid.router.http.readTimeout=PT5M + druid.router.http.numMaxThreads=100 + druid.server.http.numThreads=100 + + # Service discovery + druid.router.defaultBrokerServiceName=druid/broker + druid.router.coordinatorServiceName=druid/coordinator + + # Management proxy to coordinator / overlord: required for unified web console. + druid.router.managementProxy.enabled=true + + middlemanagers: + nodeType: "middleManager" + nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/middleManager" + druid.port: 8088 + services: + - spec: + type: NodePort + ports: + - name: middlemanager-service-port + nodePort: 30500 + port: 8088 + protocol: TCP + targetPort: 8088 + selector: + nodeSpecUniqueStr: druid-tiny-cluster-middleManagers + metadata: + name: middlemanager-%s-service + - spec: + type: ClusterIP + clusterIP: None + replicas: 1 + runtime.properties: | + druid.service=druid/middleManager + druid.worker.capacity=10 + druid.indexer.runner.javaOpts=-server -XX:MaxDirectMemorySize=10240g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/druid/data/tmp -Dlog4j.debug -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=50 -XX:GCLogFileSize=10m -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:+UseG1GC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Xloggc:/druid/data/logs/peon.gc.%t.%p.log -XX:HeapDumpPath=/druid/data/logs/peon.%t.%p.hprof -Xms256m -Xmx256m + druid.indexer.task.baseTaskDir=/druid/data/baseTaskDir + druid.server.http.numThreads=10 + druid.indexer.fork.property.druid.processing.buffer.sizeBytes=268435456 + druid.indexer.fork.property.druid.processing.numMergeBuffers=1 + druid.indexer.fork.property.druid.processing.numThreads=1 + extra.jvm.options: |- + -Xmx256m + -Xms256m + volumeMounts: + - mountPath: /druid/data + name: data-volume + volumes: + - name: data-volume + hostPath: + path: REPLACE_VOLUMES/tmp + resources: + requests: + memory: "3G" + limits: + memory: "3G" diff --git a/integration-tests/k8s_run_config_file.json b/integration-tests/k8s_run_config_file.json new file mode 100644 index 000000000000..249e44b19ca3 --- /dev/null +++ b/integration-tests/k8s_run_config_file.json @@ -0,0 +1,16 @@ +{ + "broker_host" : "localhost", + "broker_port" : "30400", + "broker_tls_url" : "http://localhost:30100", + "router_host" : "localhost", + "router_port" : "30400", + "router_tls_url" : "http://localhost:30400", + "indexer_host" : "localhost", + "indexer_port" : "30400", + "historical_host" : "localhost", + "historical_port" : "30300", + "coordinator_host" : "localhost", + "coordinator_port" : "30400", + "middlemanager_host": "localhost", + "zookeeper_hosts": "localhost:30600" +} \ No newline at end of file diff --git a/integration-tests/script/docker_compose_args.sh b/integration-tests/script/docker_compose_args.sh new file mode 100644 index 000000000000..73f3261b9c69 --- /dev/null +++ b/integration-tests/script/docker_compose_args.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# picks appropriate docker-compose argments to use when bringing up and down integration test clusters +# for a given test group +getComposeArgs() +{ + if [ -z "$DRUID_INTEGRATION_TEST_OVERRIDE_CONFIG_PATH" ] + then + # Sanity check: DRUID_INTEGRATION_TEST_INDEXER must be "indexer" or "middleManager" + if [ "$DRUID_INTEGRATION_TEST_INDEXER" != "indexer" ] && [ "$DRUID_INTEGRATION_TEST_INDEXER" != "middleManager" ] + then + echo "DRUID_INTEGRATION_TEST_INDEXER must be 'indexer' or 'middleManager' (is '$DRUID_INTEGRATION_TEST_INDEXER')" + exit 1 + fi + + if [ "$DRUID_INTEGRATION_TEST_INDEXER" = "indexer" ] + then + # Sanity check: cannot combine CliIndexer tests with security, query-retry tests + if [ "$DRUID_INTEGRATION_TEST_GROUP" = "security" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-retry" ] || [ "$DRUID_INTEGRATION_TEST_GROUP" = "high-availability" ] + then + echo "Cannot run test group '$DRUID_INTEGRATION_TEST_GROUP' with CliIndexer" + exit 1 + fi + + # Replace MiddleManager with Indexer + echo "-f ${DOCKERDIR}/docker-compose.cli-indexer.yml" + elif [ "$DRUID_INTEGRATION_TEST_GROUP" = "security" ] + then + # default + additional druid router (custom-check-tls, permissive-tls, no-client-auth-tls) + echo "-f ${DOCKERDIR}/docker-compose.yml -f ${DOCKERDIR}/docker-compose.security.yml" + elif [ "$DRUID_INTEGRATION_TEST_GROUP" = "query-retry" ] + then + # default + additional historical modified for query retry test + # See CliHistoricalForQueryRetryTest. + echo "-f ${DOCKERDIR}/docker-compose.query-retry-test.yml" + elif [ "$DRUID_INTEGRATION_TEST_GROUP" = "high-availability" ] + then + # the 'high availability' test cluster with multiple coordinators and overlords + echo "-f ${DOCKERDIR}/docker-compose.high-availability.yml" + else + # default + echo "-f ${DOCKERDIR}/docker-compose.yml" + fi + else + # with override config + echo "-f ${DOCKERDIR}/docker-compose.override-env.yml" + fi +} diff --git a/integration-tests/script/setup_druid_on_k8s.sh b/integration-tests/script/setup_druid_on_k8s.sh new file mode 100755 index 000000000000..aab018f238b1 --- /dev/null +++ b/integration-tests/script/setup_druid_on_k8s.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +export KUBECTL="sudo /usr/local/bin/kubectl" + +# setup client keystore +cd integration-tests +./docker/tls/generate-client-certs-and-keystores.sh +rm -rf docker/client_tls +cp -r client_tls docker/client_tls +cd .. + +# Build Docker images for pods +mvn -B -ff -q dependency:go-offline \ + install \ + -Pdist,bundle-contrib-exts \ + -Pskip-static-checks,skip-tests \ + -Dmaven.javadoc.skip=true + +docker build -t druid/cluster:v1 -f distribution/docker/DockerfileBuildTarAdvanced . + +# This tmp dir is used for MiddleManager pod and Historical Pod to cache segments. +mkdir tmp +chmod 777 tmp + +$KUBECTL apply -f integration-tests/k8s/role-and-binding.yaml +sed -i "s|REPLACE_VOLUMES|`pwd`|g" integration-tests/k8s/tiny-cluster.yaml +$KUBECTL apply -f integration-tests/k8s/tiny-cluster.yaml + +# Wait a bit +sleep 120 + +## Debug And FastFail + +$KUBECTL get pod +$KUBECTL get svc + diff --git a/integration-tests/script/setup_druid_operator_on_k8s.sh b/integration-tests/script/setup_druid_operator_on_k8s.sh new file mode 100755 index 000000000000..392841976634 --- /dev/null +++ b/integration-tests/script/setup_druid_operator_on_k8s.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +export DRUID_OPERATOR_VERSION=0.0.3 +export KUBECTL="sudo /usr/local/bin/kubectl" + + +# Prepare For Druid-Operator +git clone https://github.com/druid-io/druid-operator.git +cd druid-operator +git checkout -b druid-operator-$DRUID_OPERATOR_VERSION druid-operator-$DRUID_OPERATOR_VERSION +cd .. +sed -i "s|REPLACE_IMAGE|druidio/druid-operator:$DRUID_OPERATOR_VERSION|g" druid-operator/deploy/operator.yaml + +# Deploy Druid Operator and Druid CR spec +$KUBECTL create -f druid-operator/deploy/service_account.yaml +$KUBECTL create -f druid-operator/deploy/role.yaml +$KUBECTL create -f druid-operator/deploy/role_binding.yaml +$KUBECTL create -f druid-operator/deploy/crds/druid.apache.org_druids_crd.yaml +$KUBECTL create -f druid-operator/deploy/operator.yaml + +echo "Setup Druid Operator on K8S Done!" diff --git a/integration-tests/script/setup_k8s_cluster.sh b/integration-tests/script/setup_k8s_cluster.sh new file mode 100755 index 000000000000..3313def4bca3 --- /dev/null +++ b/integration-tests/script/setup_k8s_cluster.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Set Necessary ENV +export CHANGE_MINIKUBE_NONE_USER=true +export MINIKUBE_WANTUPDATENOTIFICATION=false +export MINIKUBE_WANTREPORTERRORPROMPT=false +export MINIKUBE_HOME=$HOME +export KUBECONFIG=$HOME/.kube/config + +sudo apt install -y conntrack + +# Lacunch K8S cluster +curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/v1.18.1/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/ +curl -Lo minikube https://storage.googleapis.com/minikube/releases/v1.8.1/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/ +sudo /usr/local/bin/minikube start --profile=minikube --vm-driver=none --kubernetes-version=v1.18.1 +sudo /usr/local/bin/minikube update-context + +echo "Setup K8S Cluster Done!" diff --git a/integration-tests/script/stop_k8s_cluster.sh b/integration-tests/script/stop_k8s_cluster.sh new file mode 100755 index 000000000000..6e8969bd12fc --- /dev/null +++ b/integration-tests/script/stop_k8s_cluster.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +sudo /usr/local/bin/minikube delete +sudo rm -rf `pwd`/tmp diff --git a/integration-tests/src/main/java/org/apache/druid/cli/CliCustomNodeRole.java b/integration-tests/src/main/java/org/apache/druid/cli/CliCustomNodeRole.java new file mode 100644 index 000000000000..85fa1fb61f5c --- /dev/null +++ b/integration-tests/src/main/java/org/apache/druid/cli/CliCustomNodeRole.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.cli; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Module; +import com.google.inject.name.Names; +import com.google.inject.servlet.GuiceFilter; +import io.airlift.airline.Command; +import org.apache.druid.client.coordinator.CoordinatorClient; +import org.apache.druid.discovery.NodeRole; +import org.apache.druid.guice.Jerseys; +import org.apache.druid.guice.LazySingleton; +import org.apache.druid.guice.LifecycleModule; +import org.apache.druid.guice.annotations.Json; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.server.http.SelfDiscoveryResource; +import org.apache.druid.server.initialization.ServerConfig; +import org.apache.druid.server.initialization.jetty.JettyServerInitUtils; +import org.apache.druid.server.initialization.jetty.JettyServerInitializer; +import org.apache.druid.server.security.AuthenticationUtils; +import org.apache.druid.server.security.Authenticator; +import org.apache.druid.server.security.AuthenticatorMapper; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.handler.StatisticsHandler; +import org.eclipse.jetty.servlet.DefaultServlet; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; + +import java.util.List; + +@Command( + name = CliCustomNodeRole.SERVICE_NAME, + description = "Some custom druid node role defined in an extension" +) +public class CliCustomNodeRole extends ServerRunnable +{ + private static final Logger LOG = new Logger(CliCustomNodeRole.class); + + public static final String SERVICE_NAME = "custom-node-role"; + public static final int PORT = 9301; + public static final int TLS_PORT = 9501; + + public CliCustomNodeRole() + { + super(LOG); + } + + @Override + protected List getModules() + { + return ImmutableList.of( + binder -> { + LOG.info("starting up"); + binder.bindConstant().annotatedWith(Names.named("serviceName")).to(CliCustomNodeRole.SERVICE_NAME); + binder.bindConstant().annotatedWith(Names.named("servicePort")).to(CliCustomNodeRole.PORT); + binder.bindConstant().annotatedWith(Names.named("tlsServicePort")).to(CliCustomNodeRole.TLS_PORT); + + binder.bind(CoordinatorClient.class).in(LazySingleton.class); + + binder.bind(JettyServerInitializer.class).to(CustomJettyServiceInitializer.class).in(LazySingleton.class); + LifecycleModule.register(binder, Server.class); + + bindNodeRoleAndAnnouncer( + binder, + DiscoverySideEffectsProvider.builder(new NodeRole(CliCustomNodeRole.SERVICE_NAME)).build() + ); + Jerseys.addResource(binder, SelfDiscoveryResource.class); + LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class)); + + } + ); + } + + // ugly mimic of other jetty initializers + private static class CustomJettyServiceInitializer implements JettyServerInitializer + { + private static List UNSECURED_PATHS = ImmutableList.of( + "/status/health" + ); + + private final ServerConfig serverConfig; + + @Inject + public CustomJettyServiceInitializer(ServerConfig serverConfig) + { + this.serverConfig = serverConfig; + } + + @Override + public void initialize(Server server, Injector injector) + { + final ServletContextHandler root = new ServletContextHandler(ServletContextHandler.SESSIONS); + root.addServlet(new ServletHolder(new DefaultServlet()), "/*"); + + final ObjectMapper jsonMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)); + final AuthenticatorMapper authenticatorMapper = injector.getInstance(AuthenticatorMapper.class); + + AuthenticationUtils.addSecuritySanityCheckFilter(root, jsonMapper); + + // perform no-op authorization for these resources + AuthenticationUtils.addNoopAuthenticationAndAuthorizationFilters(root, UNSECURED_PATHS); + + List authenticators = authenticatorMapper.getAuthenticatorChain(); + AuthenticationUtils.addAuthenticationFilterChain(root, authenticators); + + JettyServerInitUtils.addAllowHttpMethodsFilter(root, serverConfig.getAllowedHttpMethods()); + + JettyServerInitUtils.addExtensionFilters(root, injector); + + // Check that requests were authorized before sending responses + AuthenticationUtils.addPreResponseAuthorizationCheckFilter( + root, + authenticators, + jsonMapper + ); + + root.addFilter(GuiceFilter.class, "/*", null); + + final HandlerList handlerList = new HandlerList(); + // Do not change the order of the handlers that have already been added + for (Handler handler : server.getHandlers()) { + handlerList.addHandler(handler); + } + + handlerList.addHandler(JettyServerInitUtils.getJettyRequestLogHandler()); + + // Add Gzip handler at the very end + handlerList.addHandler( + JettyServerInitUtils.wrapWithDefaultGzipHandler( + root, + serverConfig.getInflateBufferSize(), + serverConfig.getCompressionLevel() + ) + ); + + final StatisticsHandler statisticsHandler = new StatisticsHandler(); + statisticsHandler.setHandler(handlerList); + + server.setHandler(statisticsHandler); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/druid/cli/CustomNodeRoleCommandCreator.java b/integration-tests/src/main/java/org/apache/druid/cli/CustomNodeRoleCommandCreator.java new file mode 100644 index 000000000000..90f40efb7f56 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/druid/cli/CustomNodeRoleCommandCreator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.cli; + +import io.airlift.airline.Cli; + +public class CustomNodeRoleCommandCreator implements CliCommandCreator +{ + @Override + public void addCommands(Cli.CliBuilder builder) + { + builder.withGroup("server").withCommands(CliCustomNodeRole.class); + } +} diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHttpInputSourceTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHttpInputSourceTest.java new file mode 100644 index 000000000000..69d1eeaaa94f --- /dev/null +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/ITHttpInputSourceTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.tests.indexer; + +import org.apache.druid.testing.guice.DruidTestModuleFactory; +import org.apache.druid.tests.TestNGGroup; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.io.Closeable; +import java.io.IOException; +import java.util.UUID; + +@Test(groups = TestNGGroup.INPUT_SOURCE) +@Guice(moduleFactory = DruidTestModuleFactory.class) +public class ITHttpInputSourceTest extends AbstractITBatchIndexTest +{ + private static final String INDEX_TASK = "/indexer/wikipedia_http_inputsource_task.json"; + private static final String INDEX_QUERIES_RESOURCE = "/indexer/wikipedia_http_inputsource_queries.json"; + + @Test + public void doTest() throws IOException + { + final String indexDatasource = "wikipedia_http_inputsource_test_" + UUID.randomUUID(); + try (final Closeable ignored1 = unloader(indexDatasource + config.getExtraDatasourceNameSuffix())) { + doIndexTest( + indexDatasource, + INDEX_TASK, + INDEX_QUERIES_RESOURCE, + false, + true, + true + ); + } + } +} diff --git a/integration-tests/src/test/java/org/apache/druid/tests/leadership/ITHighAvailabilityTest.java b/integration-tests/src/test/java/org/apache/druid/tests/leadership/ITHighAvailabilityTest.java new file mode 100644 index 000000000000..0cf0f4b2be11 --- /dev/null +++ b/integration-tests/src/test/java/org/apache/druid/tests/leadership/ITHighAvailabilityTest.java @@ -0,0 +1,300 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.tests.leadership; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import org.apache.druid.cli.CliCustomNodeRole; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.curator.discovery.ServerDiscoveryFactory; +import org.apache.druid.discovery.DiscoveryDruidNode; +import org.apache.druid.discovery.DruidNodeDiscovery; +import org.apache.druid.discovery.DruidNodeDiscoveryProvider; +import org.apache.druid.discovery.NodeRole; +import org.apache.druid.java.util.common.ISE; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.http.client.HttpClient; +import org.apache.druid.java.util.http.client.Request; +import org.apache.druid.java.util.http.client.response.StatusResponseHandler; +import org.apache.druid.java.util.http.client.response.StatusResponseHolder; +import org.apache.druid.testing.IntegrationTestingConfig; +import org.apache.druid.testing.clients.CoordinatorResourceTestClient; +import org.apache.druid.testing.guice.DruidTestModuleFactory; +import org.apache.druid.testing.guice.TestClient; +import org.apache.druid.testing.utils.DruidClusterAdminClient; +import org.apache.druid.testing.utils.ITRetryUtil; +import org.apache.druid.testing.utils.SqlTestQueryHelper; +import org.apache.druid.tests.TestNGGroup; +import org.apache.druid.tests.indexer.AbstractIndexerTest; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.testng.Assert; +import org.testng.annotations.Guice; +import org.testng.annotations.Test; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +@Test(groups = TestNGGroup.HIGH_AVAILABILTY) +@Guice(moduleFactory = DruidTestModuleFactory.class) +public class ITHighAvailabilityTest +{ + private static final Logger LOG = new Logger(ITHighAvailabilityTest.class); + private static final String SYSTEM_QUERIES_RESOURCE = "/queries/high_availability_sys.json"; + private static final int NUM_LEADERSHIP_SWAPS = 3; + + @Inject + private IntegrationTestingConfig config; + + @Inject + private DruidClusterAdminClient druidClusterAdminClient; + + @Inject + ServerDiscoveryFactory factory; + + @Inject + DruidNodeDiscoveryProvider druidNodeDiscovery; + + @Inject + CoordinatorResourceTestClient coordinatorClient; + + @Inject + SqlTestQueryHelper queryHelper; + + @Inject + ObjectMapper jsonMapper; + + @Inject + @TestClient + HttpClient httpClient; + + @Test + public void testLeadershipChanges() throws Exception + { + int runCount = 0; + String previousCoordinatorLeader = null; + String previousOverlordLeader = null; + // fetch current leaders, make sure queries work, then swap leaders and do it again + do { + String coordinatorLeader = getLeader("coordinator"); + String overlordLeader = getLeader("indexer"); + + // we expect leadership swap to happen + Assert.assertNotEquals(previousCoordinatorLeader, coordinatorLeader); + Assert.assertNotEquals(previousOverlordLeader, overlordLeader); + + previousCoordinatorLeader = coordinatorLeader; + previousOverlordLeader = overlordLeader; + + String queries = fillTemplate( + config, + AbstractIndexerTest.getResourceAsString(SYSTEM_QUERIES_RESOURCE), + overlordLeader, + coordinatorLeader + ); + queryHelper.testQueriesFromString(queries); + + swapLeadersAndWait(coordinatorLeader, overlordLeader); + } while (runCount++ < NUM_LEADERSHIP_SWAPS); + } + + @Test + public void testDiscoveryAndSelfDiscovery() + { + ITRetryUtil.retryUntil( + () -> { + List disco = ImmutableList.of( + druidNodeDiscovery.getForNodeRole(NodeRole.COORDINATOR), + druidNodeDiscovery.getForNodeRole(NodeRole.OVERLORD), + druidNodeDiscovery.getForNodeRole(NodeRole.HISTORICAL), + druidNodeDiscovery.getForNodeRole(NodeRole.MIDDLE_MANAGER), + druidNodeDiscovery.getForNodeRole(NodeRole.INDEXER), + druidNodeDiscovery.getForNodeRole(NodeRole.BROKER), + druidNodeDiscovery.getForNodeRole(NodeRole.ROUTER) + ); + + int servicesDiscovered = 0; + for (DruidNodeDiscovery nodeRole : disco) { + Collection nodes = nodeRole.getAllNodes(); + servicesDiscovered += testSelfDiscovery(nodes); + } + return servicesDiscovered > 5; + }, + true, + TimeUnit.SECONDS.toMillis(5), + 60, + "Standard services discovered" + ); + } + + @Test + public void testCustomDiscovery() + { + ITRetryUtil.retryUntil( + () -> { + DruidNodeDiscovery customDisco = + druidNodeDiscovery.getForNodeRole(new NodeRole(CliCustomNodeRole.SERVICE_NAME)); + int count = testSelfDiscovery(customDisco.getAllNodes()); + return count > 0; + }, + true, + TimeUnit.SECONDS.toMillis(5), + 60, + "Custom service discovered" + ); + } + + private int testSelfDiscovery(Collection nodes) + throws MalformedURLException, ExecutionException, InterruptedException + { + int count = 0; + + for (DiscoveryDruidNode node : nodes) { + final String location = StringUtils.format( + "http://%s:%s/status/selfDiscovered", + config.isDocker() ? config.getDockerHost() : node.getDruidNode().getHost(), + node.getDruidNode().getPlaintextPort() + ); + LOG.info("testing self discovery %s", location); + StatusResponseHolder response = httpClient.go( + new Request(HttpMethod.GET, new URL(location)), + StatusResponseHandler.getInstance() + ).get(); + LOG.info("%s responded with %s", location, response.getStatus().getCode()); + Assert.assertEquals(response.getStatus(), HttpResponseStatus.OK); + count++; + } + return count; + } + + private void swapLeadersAndWait(String coordinatorLeader, String overlordLeader) + { + Runnable waitUntilCoordinatorSupplier; + if (isCoordinatorOneLeader(config, coordinatorLeader)) { + druidClusterAdminClient.restartCoordinatorContainer(); + waitUntilCoordinatorSupplier = () -> druidClusterAdminClient.waitUntilCoordinatorReady(); + } else { + druidClusterAdminClient.restartCoordinatorTwoContainer(); + waitUntilCoordinatorSupplier = () -> druidClusterAdminClient.waitUntilCoordinatorTwoReady(); + } + + Runnable waitUntilOverlordSupplier; + if (isOverlordOneLeader(config, overlordLeader)) { + druidClusterAdminClient.restartOverlordContainer(); + waitUntilOverlordSupplier = () -> druidClusterAdminClient.waitUntilIndexerReady(); + } else { + druidClusterAdminClient.restartOverlordTwoContainer(); + waitUntilOverlordSupplier = () -> druidClusterAdminClient.waitUntilOverlordTwoReady(); + } + waitUntilCoordinatorSupplier.run(); + waitUntilOverlordSupplier.run(); + } + + private String getLeader(String service) + { + try { + StatusResponseHolder response = httpClient.go( + new Request( + HttpMethod.GET, + new URL(StringUtils.format( + "%s/druid/%s/v1/leader", + config.getRouterUrl(), + service + )) + ), + StatusResponseHandler.getInstance() + ).get(); + + if (!response.getStatus().equals(HttpResponseStatus.OK)) { + throw new ISE( + "Error while fetching leader from[%s] status[%s] content[%s]", + config.getRouterUrl(), + response.getStatus(), + response.getContent() + ); + } + return response.getContent(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + private static String fillTemplate(IntegrationTestingConfig config, String template, String overlordLeader, String coordinatorLeader) + { + /* + {"host":"%%BROKER%%","server_type":"broker", "is_leader": %%NON_LEADER%%}, + {"host":"%%COORDINATOR_ONE%%","server_type":"coordinator", "is_leader": %%COORDINATOR_ONE_LEADER%%}, + {"host":"%%COORDINATOR_TWO%%","server_type":"coordinator", "is_leader": %%COORDINATOR_TWO_LEADER%%}, + {"host":"%%OVERLORD_ONE%%","server_type":"overlord", "is_leader": %%OVERLORD_ONE_LEADER%%}, + {"host":"%%OVERLORD_TWO%%","server_type":"overlord", "is_leader": %%OVERLORD_TWO_LEADER%%}, + {"host":"%%ROUTER%%","server_type":"router", "is_leader": %%NON_LEADER%%} + */ + String working = template; + + working = StringUtils.replace(working, "%%OVERLORD_ONE%%", config.getOverlordInternalHost()); + working = StringUtils.replace(working, "%%OVERLORD_TWO%%", config.getOverlordTwoInternalHost()); + working = StringUtils.replace(working, "%%COORDINATOR_ONE%%", config.getCoordinatorInternalHost()); + working = StringUtils.replace(working, "%%COORDINATOR_TWO%%", config.getCoordinatorTwoInternalHost()); + working = StringUtils.replace(working, "%%BROKER%%", config.getBrokerInternalHost()); + working = StringUtils.replace(working, "%%ROUTER%%", config.getRouterInternalHost()); + if (isOverlordOneLeader(config, overlordLeader)) { + working = StringUtils.replace(working, "%%OVERLORD_ONE_LEADER%%", "1"); + working = StringUtils.replace(working, "%%OVERLORD_TWO_LEADER%%", "0"); + } else { + working = StringUtils.replace(working, "%%OVERLORD_ONE_LEADER%%", "0"); + working = StringUtils.replace(working, "%%OVERLORD_TWO_LEADER%%", "1"); + } + if (isCoordinatorOneLeader(config, coordinatorLeader)) { + working = StringUtils.replace(working, "%%COORDINATOR_ONE_LEADER%%", "1"); + working = StringUtils.replace(working, "%%COORDINATOR_TWO_LEADER%%", "0"); + } else { + working = StringUtils.replace(working, "%%COORDINATOR_ONE_LEADER%%", "0"); + working = StringUtils.replace(working, "%%COORDINATOR_TWO_LEADER%%", "1"); + } + working = StringUtils.replace(working, "%%NON_LEADER%%", String.valueOf(NullHandling.defaultLongValue())); + return working; + } + + private static boolean isCoordinatorOneLeader(IntegrationTestingConfig config, String coordinatorLeader) + { + return coordinatorLeader.contains(transformHost(config.getCoordinatorInternalHost())); + } + + private static boolean isOverlordOneLeader(IntegrationTestingConfig config, String overlordLeader) + { + return overlordLeader.contains(transformHost(config.getOverlordInternalHost())); + } + + /** + * host + ':' which should be enough to distinguish subsets, e.g. 'druid-coordinator:8081' from + * 'druid-coordinator-two:8081' for example + */ + private static String transformHost(String host) + { + return StringUtils.format("%s:", host); + } +} diff --git a/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_queries.json b/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_queries.json new file mode 100644 index 000000000000..11496c223d43 --- /dev/null +++ b/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_queries.json @@ -0,0 +1,98 @@ +[ + { + "description": "timeseries, 1 agg, all", + "query":{ + "queryType" : "timeBoundary", + "dataSource": "%%DATASOURCE%%" + }, + "expectedResults":[ + { + "timestamp" : "2016-06-27T00:00:11.000Z", + "result" : { + "minTime" : "2016-06-27T00:00:11.000Z", + "maxTime" : "2016-06-27T21:31:02.000Z" + } + } + ] + }, + { + "description": "timeseries, datasketch aggs, all", + "query":{ + "queryType" : "timeseries", + "dataSource": "%%DATASOURCE%%", + "granularity":"day", + "intervals":[ + "2016-06-27/P1D" + ], + "filter":null, + "aggregations":[ + { + "type": "HLLSketchMerge", + "name": "approxCountHLL", + "fieldName": "HLLSketchBuild", + "lgK": 12, + "tgtHllType": "HLL_4", + "round": true + }, + { + "type":"thetaSketch", + "name":"approxCountTheta", + "fieldName":"thetaSketch", + "size":16384, + "shouldFinalize":true, + "isInputThetaSketch":false, + "errorBoundsStdDev":null + }, + { + "type":"quantilesDoublesSketch", + "name":"quantilesSketch", + "fieldName":"quantilesDoublesSketch", + "k":128 + } + ] + }, + "expectedResults":[ + { + "timestamp" : "2016-06-27T00:00:00.000Z", + "result" : { + "quantilesSketch":48866, + "approxCountTheta":7862.0, + "approxCountHLL":7961 + } + } + ] + }, + { + "description": "timeseries, stringFirst/stringLast aggs, all", + "query":{ + "queryType" : "timeseries", + "dataSource": "%%DATASOURCE%%", + "granularity":"day", + "intervals":[ + "2016-06-27/P1D" + ], + "filter":null, + "aggregations":[ + { + "type": "stringFirst", + "name": "first_user", + "fieldName": "user" + }, + { + "type":"stringLast", + "name":"last_user", + "fieldName":"user" + } + ] + }, + "expectedResults":[ + { + "timestamp" : "2016-06-27T00:00:00.000Z", + "result" : { + "first_user":"Lsjbot", + "last_user":"EmausBot" + } + } + ] + } +] \ No newline at end of file diff --git a/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_task.json b/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_task.json new file mode 100644 index 000000000000..6038327d195a --- /dev/null +++ b/integration-tests/src/test/resources/indexer/wikipedia_http_inputsource_task.json @@ -0,0 +1,90 @@ +{ + "type": "index_parallel", + "spec": { + "dataSchema": { + "dataSource": "%%DATASOURCE%%", + "timestampSpec": { + "column": "timestamp" + }, + "dimensionsSpec": { + "dimensions": [ + "page", + {"type": "string", "name": "language", "createBitmapIndex": false}, + "user", + "unpatrolled", + "newPage", + "robot", + "anonymous", + "namespace", + "continent", + "country", + "region", + "city" + ] + }, + "metricsSpec": [ + { + "type": "count", + "name": "count" + }, + { + "type": "doubleSum", + "name": "added", + "fieldName": "added" + }, + { + "type": "doubleSum", + "name": "deleted", + "fieldName": "deleted" + }, + { + "type": "doubleSum", + "name": "delta", + "fieldName": "delta" + }, + { + "name": "thetaSketch", + "type": "thetaSketch", + "fieldName": "user" + }, + { + "name": "quantilesDoublesSketch", + "type": "quantilesDoublesSketch", + "fieldName": "delta" + }, + { + "name": "HLLSketchBuild", + "type": "HLLSketchBuild", + "fieldName": "user" + } + ], + "granularitySpec": { + "segmentGranularity": "DAY", + "queryGranularity": "second", + "intervals" : [ "2016-06/P1M" ] + } + }, + "ioConfig": { + "type": "index_parallel", + "inputSource": { + "type": "http", + "uris": ["https://druid.apache.org/data/wikipedia.json.gz", "https://druid.apache.org/data/wikipedia.json.gz"] + }, + "inputFormat": { + "type": "json" + } + }, + "tuningConfig": { + "type": "index_parallel", + "maxNumConcurrentSubTasks": 10, + "partitionsSpec": { + "type": "hashed" + }, + "forceGuaranteedRollup": true, + "splitHintSpec": { + "type": "maxSize", + "maxNumFiles": 1 + } + } + } +} \ No newline at end of file diff --git a/integration-tests/src/test/resources/indexer/wikipedia_index_with_merge_column_limit_task.json b/integration-tests/src/test/resources/indexer/wikipedia_index_with_merge_column_limit_task.json new file mode 100644 index 000000000000..35b115c9f191 --- /dev/null +++ b/integration-tests/src/test/resources/indexer/wikipedia_index_with_merge_column_limit_task.json @@ -0,0 +1,86 @@ +{ + "type": "index", + "spec": { + "dataSchema": { + "dataSource": "%%DATASOURCE%%", + "metricsSpec": [ + { + "type": "count", + "name": "count" + }, + { + "type": "doubleSum", + "name": "added", + "fieldName": "added" + }, + { + "type": "doubleSum", + "name": "deleted", + "fieldName": "deleted" + }, + { + "type": "doubleSum", + "name": "delta", + "fieldName": "delta" + }, + { + "name": "thetaSketch", + "type": "thetaSketch", + "fieldName": "user" + }, + { + "name": "quantilesDoublesSketch", + "type": "quantilesDoublesSketch", + "fieldName": "delta" + }, + { + "name": "HLLSketchBuild", + "type": "HLLSketchBuild", + "fieldName": "user" + } + ], + "granularitySpec": { + "segmentGranularity": "DAY", + "queryGranularity": "second", + "intervals" : [ "2013-08-31/2013-09-02" ] + }, + "parser": { + "parseSpec": { + "format" : "json", + "timestampSpec": { + "column": "timestamp" + }, + "dimensionsSpec": { + "dimensions": [ + "page", + {"type": "string", "name": "language", "createBitmapIndex": false}, + "user", + "unpatrolled", + "newPage", + "robot", + "anonymous", + "namespace", + "continent", + "country", + "region", + "city" + ] + } + } + } + }, + "ioConfig": { + "type": "index", + "firehose": { + "type": "local", + "baseDir": "/resources/data/batch_index/json", + "filter": "wikipedia_index_data*" + } + }, + "tuningConfig": { + "type": "index", + "maxRowsPerSegment": 3, + "maxColumnsToMerge" : 30 + } + } +} \ No newline at end of file diff --git a/integration-tests/src/test/resources/queries/broadcast_join_after_drop_metadata_queries.json b/integration-tests/src/test/resources/queries/broadcast_join_after_drop_metadata_queries.json new file mode 100644 index 000000000000..26e36885c6fe --- /dev/null +++ b/integration-tests/src/test/resources/queries/broadcast_join_after_drop_metadata_queries.json @@ -0,0 +1,9 @@ +[ + { + "description": "query information schema to make sure datasource is joinable and broadcast", + "query": { + "query": "SELECT TABLE_NAME, IS_JOINABLE, IS_BROADCAST FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = '%%JOIN_DATASOURCE%%' AND IS_JOINABLE = 'YES' AND IS_BROADCAST = 'YES' AND TABLE_SCHEMA = 'druid'" + }, + "expectedResults": [] + } +] \ No newline at end of file diff --git a/integration-tests/src/test/resources/queries/high_availability_sys.json b/integration-tests/src/test/resources/queries/high_availability_sys.json new file mode 100644 index 000000000000..d5d60d4f2979 --- /dev/null +++ b/integration-tests/src/test/resources/queries/high_availability_sys.json @@ -0,0 +1,39 @@ +[ + { + "description": "query sys.servers to make sure all expected servers are available", + "query": { + "query": "SELECT host, server_type, is_leader FROM sys.servers ORDER BY host" + }, + "expectedResults": [ + {"host":"%%BROKER%%","server_type":"broker", "is_leader": %%NON_LEADER%%}, + {"host":"%%COORDINATOR_ONE%%","server_type":"coordinator", "is_leader": %%COORDINATOR_ONE_LEADER%%}, + {"host":"%%COORDINATOR_TWO%%","server_type":"coordinator", "is_leader": %%COORDINATOR_TWO_LEADER%%}, + {"host":"%%OVERLORD_ONE%%","server_type":"overlord", "is_leader": %%OVERLORD_ONE_LEADER%%}, + {"host":"%%OVERLORD_TWO%%","server_type":"overlord", "is_leader": %%OVERLORD_TWO_LEADER%%}, + {"host":"%%ROUTER%%","server_type":"router", "is_leader": %%NON_LEADER%%} + ] + }, + { + "description": "query sys.segments which is fed via coordinator data", + "query": { + "query": "SELECT datasource, count(*) FROM sys.segments WHERE datasource='wikipedia_editstream' OR datasource='twitterstream' GROUP BY 1 " + }, + "expectedResults": [ + { + "datasource": "wikipedia_editstream", + "EXPR$1": 1 + }, + { + "datasource": "twitterstream", + "EXPR$1": 3 + } + ] + }, + { + "description": "query sys.tasks which is fed via overlord", + "query": { + "query": "SELECT datasource, count(*) FROM sys.tasks WHERE datasource='wikipedia_editstream' OR datasource='twitterstream' GROUP BY 1 " + }, + "expectedResults": [] + } +] \ No newline at end of file diff --git a/licenses/bin/json-bigint-native.MIT b/licenses/bin/json-bigint-native.MIT new file mode 100644 index 000000000000..9776c8164bb4 --- /dev/null +++ b/licenses/bin/json-bigint-native.MIT @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2020 Vadim Ogievetsky, Andrey Sidorov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/processing/src/main/java/org/apache/druid/query/BadJsonQueryException.java b/processing/src/main/java/org/apache/druid/query/BadJsonQueryException.java new file mode 100644 index 000000000000..8be18edf18a0 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/BadJsonQueryException.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonParseException; + +public class BadJsonQueryException extends BadQueryException +{ + public static final String ERROR_CODE = "Json parse failed"; + public static final String ERROR_CLASS = JsonParseException.class.getName(); + + public BadJsonQueryException(JsonParseException e) + { + this(ERROR_CODE, e.getMessage(), ERROR_CLASS); + } + + @JsonCreator + private BadJsonQueryException( + @JsonProperty("error") String errorCode, + @JsonProperty("errorMessage") String errorMessage, + @JsonProperty("errorClass") String errorClass + ) + { + super(errorCode, errorMessage, errorClass); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/BadQueryException.java b/processing/src/main/java/org/apache/druid/query/BadQueryException.java new file mode 100644 index 000000000000..b115cc1170c3 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/BadQueryException.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query; + +/** + * An abstract class for all query exceptions that should return a bad request status code (400). + * + * See {@code BadRequestException} for non-query requests. + */ +public abstract class BadQueryException extends QueryException +{ + public static final int STATUS_CODE = 400; + + protected BadQueryException(String errorCode, String errorMessage, String errorClass) + { + super(errorCode, errorMessage, errorClass, null); + } + + protected BadQueryException(String errorCode, String errorMessage, String errorClass, String host) + { + super(errorCode, errorMessage, errorClass, host); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/GroupingAggregatorFactory.java b/processing/src/main/java/org/apache/druid/query/aggregation/GroupingAggregatorFactory.java new file mode 100644 index 000000000000..62fbb47aa095 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/GroupingAggregatorFactory.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import org.apache.druid.annotations.EverythingIsNonnullByDefault; +import org.apache.druid.query.aggregation.constant.LongConstantAggregator; +import org.apache.druid.query.aggregation.constant.LongConstantBufferAggregator; +import org.apache.druid.query.aggregation.constant.LongConstantVectorAggregator; +import org.apache.druid.query.cache.CacheKeyBuilder; +import org.apache.druid.segment.ColumnInspector; +import org.apache.druid.segment.ColumnSelectorFactory; +import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.vector.VectorColumnSelectorFactory; +import org.apache.druid.utils.CollectionUtils; + +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * This class implements {@code grouping} function to determine the grouping that a row is part of. Different result rows + * for a query could have different grouping columns when subtotals are used. + * + * This aggregator factory takes following arguments + * - {@code name} - Name of aggregators + * - {@code groupings} - List of dimensions that the user is interested in tracking + * - {@code keyDimensions} - The list of grouping dimensions being included in the result row. This list is a subset of + * {@code groupings}. This argument cannot be passed by the user. It is set by druid engine + * when a particular subtotal spec is being processed. Whenever druid engine processes a new + * subtotal spec, engine sets that subtotal spec as new {@code keyDimensions}. + * + * When key dimensions are updated, {@code value} is updated as well. How the value is determined is captured + * at {@link #groupingId(List, Set)}. + * + * since grouping has to be calculated only once, it could have been implemented as a virtual function or + * post-aggregator etc. We modelled it as an aggregation operator so that its output can be used in a post-aggregator. + * Calcite too models grouping function as an aggregation operator. + * Since it is a non-trivial special aggregation, implementing it required changes in core druid engine to work. There + * were few approaches. We chose the approach that required least changes in core druid. + * Refer to https://github.com/apache/druid/pull/10518#discussion_r532941216 for more details. + * + * Currently, it works in following way + * - On data servers (no change), + * - this factory generates {@link LongConstantAggregator} / {@link LongConstantBufferAggregator} / {@link LongConstantVectorAggregator} + * with keyDimensions as null + * - The aggregators don't actually aggregate anything and their result is not actually used. We could have removed + * these aggregators on data servers but that would result in a signature mismatch on broker and data nodes. That requires + * extra handling and is error-prone. + * - On brokers + * - Results from data node is already being re-processed for each subtotal spec. We made modifications in this path to update the + * grouping id for each row. + * + */ +@EverythingIsNonnullByDefault +public class GroupingAggregatorFactory extends AggregatorFactory +{ + private static final Comparator VALUE_COMPARATOR = Long::compare; + private final String name; + private final List groupings; + private final long value; + @Nullable + private final Set keyDimensions; + + @JsonCreator + public GroupingAggregatorFactory( + @JsonProperty("name") String name, + @JsonProperty("groupings") List groupings + ) + { + this(name, groupings, null); + } + + @VisibleForTesting + GroupingAggregatorFactory( + String name, + List groupings, + @Nullable Set keyDimensions + ) + { + Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); + this.name = name; + this.groupings = groupings; + this.keyDimensions = keyDimensions; + value = groupingId(groupings, keyDimensions); + } + + @Override + public Aggregator factorize(ColumnSelectorFactory metricFactory) + { + return new LongConstantAggregator(value); + } + + @Override + public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) + { + return new LongConstantBufferAggregator(value); + } + + @Override + public VectorAggregator factorizeVector(VectorColumnSelectorFactory selectorFactory) + { + return new LongConstantVectorAggregator(value); + } + + @Override + public boolean canVectorize(ColumnInspector columnInspector) + { + return true; + } + + /** + * Replace the param {@code keyDimensions} with the new set of key dimensions + */ + public GroupingAggregatorFactory withKeyDimensions(Set newKeyDimensions) + { + return new GroupingAggregatorFactory(name, groupings, newKeyDimensions); + } + + @Override + public Comparator getComparator() + { + return VALUE_COMPARATOR; + } + + @JsonProperty + public List getGroupings() + { + return groupings; + } + + @Override + @JsonProperty + public String getName() + { + return name; + } + + public long getValue() + { + return value; + } + + @Nullable + @Override + public Object combine(@Nullable Object lhs, @Nullable Object rhs) + { + if (null == lhs) { + return rhs; + } + return lhs; + } + + @Override + public AggregatorFactory getCombiningFactory() + { + return new GroupingAggregatorFactory(name, groupings, keyDimensions); + } + + @Override + public List getRequiredColumns() + { + return Collections.singletonList(new GroupingAggregatorFactory(name, groupings, keyDimensions)); + } + + @Override + public Object deserialize(Object object) + { + return object; + } + + @Nullable + @Override + public Object finalizeComputation(@Nullable Object object) + { + return object; + } + + @Override + public List requiredFields() + { + // The aggregator doesn't need to read any fields. + return Collections.emptyList(); + } + + @Override + public ValueType getType() + { + return ValueType.LONG; + } + + @Override + public ValueType getFinalizedType() + { + return ValueType.LONG; + } + + @Override + public int getMaxIntermediateSize() + { + return Long.BYTES; + } + + @Override + public byte[] getCacheKey() + { + CacheKeyBuilder keyBuilder = new CacheKeyBuilder(AggregatorUtil.GROUPING_CACHE_TYPE_ID) + .appendStrings(groupings); + if (null != keyDimensions) { + keyBuilder.appendStrings(keyDimensions); + } + return keyBuilder.build(); + } + + /** + * Given the list of grouping dimensions, returns a long value where each bit at position X in the returned value + * corresponds to the dimension in groupings at same position X. X is the position relative to the right end. if + * keyDimensions contain the grouping dimension at position X, the bit is set to 0 at position X, otherwise it is + * set to 1. + * + * groupings keyDimensions value (3 least significant bits) value (long) + * a,b,c [a] 011 3 + * a,b,c [b] 101 5 + * a,b,c [c] 110 6 + * a,b,c [a,b] 001 1 + * a,b,c [a,c] 010 2 + * a,b,c [b,c] 100 4 + * a,b,c [a,b,c] 000 0 + * a,b,c [] 111 7 // None included + * a,b,c 000 0 // All included + */ + private long groupingId(List groupings, @Nullable Set keyDimensions) + { + Preconditions.checkArgument(!CollectionUtils.isNullOrEmpty(groupings), "Must have a non-empty grouping dimensions"); + // (Long.SIZE - 1) is just a sanity check. In practice, it will be just few dimensions. This limit + // also makes sure that values are always positive. + Preconditions.checkArgument( + groupings.size() < Long.SIZE, + "Number of dimensions %s is more than supported %s", + groupings.size(), + Long.SIZE - 1 + ); + long temp = 0L; + for (String groupingDimension : groupings) { + temp = temp << 1; + if (!isDimensionIncluded(groupingDimension, keyDimensions)) { + temp = temp | 1L; + } + } + return temp; + } + + private boolean isDimensionIncluded(String dimToCheck, @Nullable Set keyDimensions) + { + if (null == keyDimensions) { + // All dimensions are included + return true; + } else { + return keyDimensions.contains(dimToCheck); + } + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + GroupingAggregatorFactory factory = (GroupingAggregatorFactory) o; + return name.equals(factory.name) && + groupings.equals(factory.groupings) && + Objects.equals(keyDimensions, factory.keyDimensions); + } + + @Override + public int hashCode() + { + return Objects.hash(name, groupings, keyDimensions); + } + + @Override + public String toString() + { + return "GroupingAggregatorFactory{" + + "name='" + name + '\'' + + ", groupings=" + groupings + + ", keyDimensions=" + keyDimensions + + '}'; + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantAggregator.java new file mode 100644 index 000000000000..1fae2715b1d4 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantAggregator.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.constant; + +import org.apache.druid.query.aggregation.Aggregator; + +/** + * This aggregator is a no-op aggregator with a fixed non-null output value. It can be used in scenarios where + * result is constant such as {@link org.apache.druid.query.aggregation.GroupingAggregatorFactory} + */ +public class LongConstantAggregator implements Aggregator +{ + private final long value; + + public LongConstantAggregator(long value) + { + this.value = value; + } + + @Override + public void aggregate() + { + // No-op + } + + @Override + public Object get() + { + return value; + } + + @Override + public float getFloat() + { + return (float) value; + } + + @Override + public long getLong() + { + return value; + } + + @Override + public void close() + { + + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantBufferAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantBufferAggregator.java new file mode 100644 index 000000000000..1ddf11b57d7b --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantBufferAggregator.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.constant; + +import org.apache.druid.query.aggregation.BufferAggregator; + +import java.nio.ByteBuffer; + +/** + * {@link BufferAggregator} variant of {@link LongConstantAggregator} + */ +public class LongConstantBufferAggregator implements BufferAggregator +{ + private final long value; + + public LongConstantBufferAggregator(long value) + { + this.value = value; + } + + @Override + public void init(ByteBuffer buf, int position) + { + // Since we always return a constant value despite what is in the buffer, there is no need to + // update the buffer at all + } + + @Override + public void aggregate(ByteBuffer buf, int position) + { + + } + + @Override + public Object get(ByteBuffer buf, int position) + { + return value; + } + + @Override + public float getFloat(ByteBuffer buf, int position) + { + return (float) value; + } + + @Override + public long getLong(ByteBuffer buf, int position) + { + return value; + } + + @Override + public void close() + { + + } +} diff --git a/processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantVectorAggregator.java b/processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantVectorAggregator.java new file mode 100644 index 000000000000..4af4b8a9fe77 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/aggregation/constant/LongConstantVectorAggregator.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.constant; + +import org.apache.druid.query.aggregation.VectorAggregator; + +import javax.annotation.Nullable; +import java.nio.ByteBuffer; + +/** + * {@link VectorAggregator} variant of {@link LongConstantAggregator} + */ +public class LongConstantVectorAggregator implements VectorAggregator +{ + private final long value; + + public LongConstantVectorAggregator(long value) + { + this.value = value; + } + + @Override + public void init(ByteBuffer buf, int position) + { + // Since we always return a constant value despite what is in the buffer, there is no need to + // update the buffer at all + } + + @Override + public void aggregate(ByteBuffer buf, int position, int startRow, int endRow) + { + + } + + @Override + public void aggregate(ByteBuffer buf, int numRows, int[] positions, @Nullable int[] rows, int positionOffset) + { + + } + + @Override + public Object get(ByteBuffer buf, int position) + { + return value; + } + + @Override + public void close() + { + + } +} diff --git a/processing/src/main/java/org/apache/druid/segment/DimensionHandlerProvider.java b/processing/src/main/java/org/apache/druid/segment/DimensionHandlerProvider.java new file mode 100644 index 000000000000..2c98e08dd2d4 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/DimensionHandlerProvider.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment; + +public interface DimensionHandlerProvider + , EncodedKeyComponentType, ActualType extends Comparable> +{ + DimensionHandler get(String dimensionName); +} diff --git a/processing/src/main/java/org/apache/druid/segment/vector/ReadableVectorInspector.java b/processing/src/main/java/org/apache/druid/segment/vector/ReadableVectorInspector.java new file mode 100644 index 000000000000..1d97087f8ba6 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/segment/vector/ReadableVectorInspector.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.vector; + +/** + * Vector inspector that can supply a unique identifier of the vector to use with caching in addition to + * sizing information + */ +public interface ReadableVectorInspector extends VectorSizeInspector +{ + /** + * A marker value that will never be returned by "getId". + */ + int NULL_ID = -1; + + /** + * Returns an integer that uniquely identifies the current vector. This is useful for caching: it is safe to assume + * nothing has changed in the vector so long as the id remains the same. + */ + int getId(); +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/GroupingAggregatorFactoryTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/GroupingAggregatorFactoryTest.java new file mode 100644 index 000000000000..0117bf1e82a5 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/GroupingAggregatorFactoryTest.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation; + +import com.google.common.collect.Sets; +import junitparams.converters.Nullable; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.query.aggregation.constant.LongConstantAggregator; +import org.apache.druid.query.aggregation.constant.LongConstantBufferAggregator; +import org.apache.druid.query.aggregation.constant.LongConstantVectorAggregator; +import org.apache.druid.segment.ColumnSelectorFactory; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.Collection; + +@RunWith(Enclosed.class) +public class GroupingAggregatorFactoryTest +{ + public static GroupingAggregatorFactory makeFactory(String[] groupings, @Nullable String[] keyDims) + { + GroupingAggregatorFactory factory = new GroupingAggregatorFactory("name", Arrays.asList(groupings)); + if (null != keyDims) { + factory = factory.withKeyDimensions(Sets.newHashSet(keyDims)); + } + return factory; + } + + public static class NewAggregatorTests + { + private ColumnSelectorFactory metricFactory; + + @Before + public void setup() + { + metricFactory = EasyMock.mock(ColumnSelectorFactory.class); + } + + @Test + public void testNewAggregator() + { + GroupingAggregatorFactory factory = makeFactory(new String[]{"a", "b"}, new String[]{"a"}); + Aggregator aggregator = factory.factorize(metricFactory); + Assert.assertEquals(LongConstantAggregator.class, aggregator.getClass()); + Assert.assertEquals(1, aggregator.getLong()); + } + + @Test + public void testNewBufferAggregator() + { + GroupingAggregatorFactory factory = makeFactory(new String[]{"a", "b"}, new String[]{"a"}); + BufferAggregator aggregator = factory.factorizeBuffered(metricFactory); + Assert.assertEquals(LongConstantBufferAggregator.class, aggregator.getClass()); + Assert.assertEquals(1, aggregator.getLong(null, 0)); + } + + @Test + public void testNewVectorAggregator() + { + GroupingAggregatorFactory factory = makeFactory(new String[]{"a", "b"}, new String[]{"a"}); + Assert.assertTrue(factory.canVectorize(metricFactory)); + VectorAggregator aggregator = factory.factorizeVector(null); + Assert.assertEquals(LongConstantVectorAggregator.class, aggregator.getClass()); + Assert.assertEquals(1L, aggregator.get(null, 0)); + } + + @Test + public void testWithKeyDimensions() + { + GroupingAggregatorFactory factory = makeFactory(new String[]{"a", "b"}, new String[]{"a"}); + Aggregator aggregator = factory.factorize(metricFactory); + Assert.assertEquals(1, aggregator.getLong()); + factory = factory.withKeyDimensions(Sets.newHashSet("b")); + aggregator = factory.factorize(metricFactory); + Assert.assertEquals(2, aggregator.getLong()); + } + } + + public static class GroupingDimensionsTest + { + @Rule + public ExpectedException exception = ExpectedException.none(); + + @Test + public void testFactory_nullGroupingDimensions() + { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Must have a non-empty grouping dimensions"); + GroupingAggregatorFactory factory = new GroupingAggregatorFactory("name", null, Sets.newHashSet("b")); + } + + @Test + public void testFactory_emptyGroupingDimensions() + { + exception.expect(IllegalArgumentException.class); + exception.expectMessage("Must have a non-empty grouping dimensions"); + makeFactory(new String[0], null); + } + + @Test + public void testFactory_highNumberOfGroupingDimensions() + { + exception.expect(IllegalArgumentException.class); + exception.expectMessage(StringUtils.format( + "Number of dimensions %d is more than supported %d", + Long.SIZE, + Long.SIZE - 1 + )); + makeFactory(new String[Long.SIZE], null); + } + } + + @RunWith(Parameterized.class) + public static class ValueTests + { + private final GroupingAggregatorFactory factory; + private final long value; + + public ValueTests(String[] groupings, @Nullable String[] keyDimensions, long value) + { + factory = makeFactory(groupings, keyDimensions); + this.value = value; + } + + @Parameterized.Parameters + public static Collection arguments() + { + String[] maxGroupingList = new String[Long.SIZE - 1]; + for (int i = 0; i < maxGroupingList.length; i++) { + maxGroupingList[i] = String.valueOf(i); + } + return Arrays.asList(new Object[][]{ + {new String[]{"a", "b"}, new String[0], 3}, + {new String[]{"a", "b"}, null, 0}, + {new String[]{"a", "b"}, new String[]{"a"}, 1}, + {new String[]{"a", "b"}, new String[]{"b"}, 2}, + {new String[]{"a", "b"}, new String[]{"a", "b"}, 0}, + {new String[]{"b", "a"}, new String[]{"a"}, 2}, + {maxGroupingList, null, 0}, + {maxGroupingList, new String[0], Long.MAX_VALUE} + }); + } + + @Test + public void testValue() + { + Assert.assertEquals(value, factory.factorize(null).getLong()); + } + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantAggregatorTest.java new file mode 100644 index 000000000000..d4f8b02220bf --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantAggregatorTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.constant; + +import org.apache.commons.lang.math.RandomUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class LongConstantAggregatorTest +{ + private long randomVal; + private LongConstantAggregator aggregator; + + @Before + public void setup() + { + randomVal = RandomUtils.nextLong(); + aggregator = new LongConstantAggregator(randomVal); + } + + @Test + public void testLong() + { + Assert.assertEquals(randomVal, aggregator.getLong()); + } + + @Test + public void testAggregate() + { + aggregator.aggregate(); + Assert.assertEquals(randomVal, aggregator.getLong()); + } + + @Test + public void testFloat() + { + Assert.assertEquals((float) randomVal, aggregator.getFloat(), 0.0001f); + } + + @Test + public void testGet() + { + Assert.assertEquals(randomVal, aggregator.get()); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantBufferAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantBufferAggregatorTest.java new file mode 100644 index 000000000000..0608fca85b74 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantBufferAggregatorTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.constant; + +import org.apache.commons.lang.math.RandomUtils; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.nio.ByteBuffer; + +public class LongConstantBufferAggregatorTest +{ + private long randomVal; + private LongConstantBufferAggregator aggregator; + private ByteBuffer byteBuffer; + + @Before + public void setup() + { + randomVal = RandomUtils.nextLong(); + aggregator = new LongConstantBufferAggregator(randomVal); + byteBuffer = EasyMock.mock(ByteBuffer.class); + EasyMock.replay(byteBuffer); + EasyMock.verifyUnexpectedCalls(byteBuffer); + } + + @Test + public void testLong() + { + Assert.assertEquals(randomVal, aggregator.getLong(byteBuffer, 0)); + } + + @Test + public void testAggregate() + { + aggregator.aggregate(byteBuffer, 0); + Assert.assertEquals(randomVal, aggregator.getLong(byteBuffer, 0)); + } + + @Test + public void testFloat() + { + Assert.assertEquals((float) randomVal, aggregator.getFloat(byteBuffer, 0), 0.0001f); + } + + @Test + public void testGet() + { + Assert.assertEquals(randomVal, aggregator.get(byteBuffer, 0)); + } +} diff --git a/processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantVectorAggregatorTest.java b/processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantVectorAggregatorTest.java new file mode 100644 index 000000000000..f62dd0369c61 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/aggregation/constant/LongConstantVectorAggregatorTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.aggregation.constant; + +import org.apache.commons.lang.math.RandomUtils; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.nio.ByteBuffer; + +public class LongConstantVectorAggregatorTest +{ + private long randomVal; + private LongConstantVectorAggregator aggregator; + private ByteBuffer byteBuffer; + + @Before + public void setup() + { + randomVal = RandomUtils.nextLong(); + aggregator = new LongConstantVectorAggregator(randomVal); + byteBuffer = EasyMock.mock(ByteBuffer.class); + EasyMock.replay(byteBuffer); + EasyMock.verifyUnexpectedCalls(byteBuffer); + } + + @Test + public void testAggregate() + { + aggregator.aggregate(byteBuffer, 0, 1, 10); + Assert.assertEquals(randomVal, aggregator.get(byteBuffer, 0)); + } + + @Test + public void testAggregateWithIndirection() + { + aggregator.aggregate(byteBuffer, 2, new int[]{2, 3}, null, 0); + Assert.assertEquals(randomVal, aggregator.get(byteBuffer, 0)); + } + + @Test + public void testGet() + { + Assert.assertEquals(randomVal, aggregator.get(byteBuffer, 0)); + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/DimensionHandlerUtilsTest.java b/processing/src/test/java/org/apache/druid/segment/DimensionHandlerUtilsTest.java new file mode 100644 index 000000000000..ee600feae662 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/DimensionHandlerUtilsTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment; + +import org.apache.druid.segment.column.ColumnCapabilitiesImpl; +import org.apache.druid.segment.column.ValueType; +import org.junit.Assert; +import org.junit.Test; + +public class DimensionHandlerUtilsTest +{ + @Test + public void testRegisterDimensionHandlerProvider() + { + DimensionHandlerUtils.registerDimensionHandlerProvider( + "testType", + d -> new DoubleDimensionHandler(d) + ); + + DimensionHandler dimensionHandler = DimensionHandlerUtils.getHandlerFromCapabilities( + "dim", + new ColumnCapabilitiesImpl().setType(ValueType.COMPLEX).setComplexTypeName("testType"), + null + ); + + Assert.assertEquals("dim", dimensionHandler.getDimensionName()); + Assert.assertTrue(dimensionHandler instanceof DoubleDimensionHandler); + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/column/ColumnBuilderTest.java b/processing/src/test/java/org/apache/druid/segment/column/ColumnBuilderTest.java new file mode 100644 index 000000000000..79733b3b7616 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/column/ColumnBuilderTest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.column; + +import org.junit.Assert; +import org.junit.Test; + +public class ColumnBuilderTest +{ + @Test + public void testSetComplexTypeName() + { + ColumnHolder holder = new ColumnBuilder() + .setType(ValueType.COMPLEX) + .setComplexTypeName("testType") + .build(); + + Assert.assertEquals("testType", holder.getCapabilities().getComplexTypeName()); + + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexCreator.java b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexCreator.java new file mode 100644 index 000000000000..3f112cc155fd --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/incremental/IncrementalIndexCreator.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.incremental; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.cfg.MapperConfig; +import com.fasterxml.jackson.databind.introspect.AnnotatedClass; +import com.fasterxml.jackson.databind.introspect.AnnotatedClassResolver; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.jsontype.SubtypeResolver; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.io.Closer; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * This class handles the incremental-index lifecycle for testing. + * Any index created using this class during the test will be closed automatically once this class is closed. + * + * To allow testing multiple incremental-index implementations, this class can be instantiated with any + * {@code AppendableIndexSpec} instance. + * Alternatively, this class can instantiate an {@code AppendableIndexSpec} for you given the appendable-index type as + * a string. + * This allows tests' parameterization with the appendable-index types as strings. + * + * To further facilitate the tests' parameterization, this class supports listing all the available incremental-index + * implementations, and produce a cartesian product of many parameter options together with each incremental-index + * implementation. + */ +public class IncrementalIndexCreator implements Closeable +{ + public static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper(); + + /** + * Allows adding support for testing unregistered indexes. + * It is used by Druid's extensions for the incremental-index. + * + * @param c an index spec class + * @param name an index spec name + */ + public static void addIndexSpec(Class c, String name) + { + JSON_MAPPER.registerSubtypes(new NamedType(c, name)); + } + + static { + // The off-heap incremental-index is not registered for production, but we want to include it in the tests. + IncrementalIndexCreator.addIndexSpec(OffheapIncrementalIndexTestSpec.class, OffheapIncrementalIndexTestSpec.TYPE); + } + + /** + * Fetch all the available incremental-index implementations. + * It can be used to parametrize the test. If more parameters are needed, use indexTypeCartesianProduct(). + * @see #indexTypeCartesianProduct(Collection[]). + * + * @return a list of all the incremental-index implementations types (String) + */ + public static List getAppendableIndexTypes() + { + SubtypeResolver resolver = JSON_MAPPER.getSubtypeResolver(); + MapperConfig config = JSON_MAPPER.getDeserializationConfig(); + AnnotatedClass cls = AnnotatedClassResolver.resolveWithoutSuperTypes(config, AppendableIndexSpec.class); + Collection types = resolver.collectAndResolveSubtypesByClass(config, cls); + return types.stream().map(NamedType::getName).filter(Objects::nonNull).distinct().collect(Collectors.toList()); + } + + public interface IndexCreator + { + /** + * Build an index given a builder and args. + * + * @param builder an incremental index builder supplied by the framework + * @param args a list of arguments that are used to configure the builder + * @return a new instance of an incremental-index + */ + IncrementalIndex createIndex(AppendableIndexBuilder builder, Object... args); + } + + private final Closer closer = Closer.create(); + + private final AppendableIndexSpec appendableIndexSpec; + + private final IndexCreator indexCreator; + + /** + * Initialize the creator. + * + * @param spec a spec that can generate a incremental-index builder + * @param indexCreator a function that generate an index given a builder and arguments + */ + public IncrementalIndexCreator(AppendableIndexSpec spec, IndexCreator indexCreator) + { + this.appendableIndexSpec = spec; + this.indexCreator = indexCreator; + } + + /** + * Initialize the creator. + * + * @param indexType an index type (name) + * @param indexCreator a function that generate an index given a builder and arguments + */ + public IncrementalIndexCreator(String indexType, IndexCreator indexCreator) throws JsonProcessingException + { + this(parseIndexType(indexType), indexCreator); + } + + /** + * Generate an AppendableIndexSpec from index type. + * + * @param indexType an index type + * @return AppendableIndexSpec instance of this type + * @throws JsonProcessingException if failed to to parse the index + */ + public static AppendableIndexSpec parseIndexType(String indexType) throws JsonProcessingException + { + return JSON_MAPPER.readValue( + StringUtils.format("{\"type\": \"%s\"}", indexType), + AppendableIndexSpec.class + ); + } + + /** + * Create an index given the input args. + * + * @param args The arguments for the index-generator + * @return An incremental-index instance + */ + public final IncrementalIndex createIndex(Object... args) + { + return createIndex(indexCreator, args); + } + + /** + * Create an index given the input args with a specialized index-creator. + * + * @param args The arguments for the index-generator + * @return An incremental-index instance + */ + public final IncrementalIndex createIndex(IndexCreator indexCreator, Object... args) + { + return closer.register(indexCreator.createIndex(appendableIndexSpec.builder(), args)); + } + + @Override + public void close() throws IOException + { + closer.close(); + + if (appendableIndexSpec instanceof Closeable) { + ((Closeable) appendableIndexSpec).close(); + } + } + + /** + * Generates all the permutations of the parameters with each of the registered appendable index types. + * It is used to parameterize the tests with all the permutations of the parameters + * together with all the appnedbale index types. + * + * For example, for a parameterized test with the following constrctor: + * {@code + * public IncrementalIndexTest(String indexType, String mode, boolean deserializeComplexMetrics) + * { + * ... + * } + * } + * + * we can test all the input combinations as follows: + * {@code + * @Parameterized.Parameters(name = "{index}: {0}, {1}, deserialize={2}") + * public static Collection constructorFeeder() + * { + * return IncrementalIndexCreator.indexTypeCartesianProduct( + * ImmutableList.of("rollup", "plain"), + * ImmutableList.of(true, false) + * ); + * } + * } + * + * @param c a list of collections of parameters + * @return the cartesian product of all parameters and appendable index types + */ + public static List indexTypeCartesianProduct(Collection... c) + { + Collection[] args = new Collection[c.length + 1]; + args[0] = getAppendableIndexTypes(); + System.arraycopy(c, 0, args, 1, c.length); + return cartesianProduct(args); + } + + /** + * Generates all the permutations of the parameters. + * + * @param c a list of collections of parameters + * @return the cartesian product of all parameters + */ + private static List cartesianProduct(Collection... c) + { + final ArrayList res = new ArrayList<>(); + final int curLength = c.length; + + if (curLength == 0) { + res.add(new Object[0]); + return res; + } + + final int curItem = curLength - 1; + for (Object[] objList : cartesianProduct(Arrays.copyOfRange(c, 0, curItem))) { + for (Object o : c[curItem]) { + Object[] newObjList = Arrays.copyOf(objList, curLength); + newObjList[curItem] = o; + res.add(newObjList); + } + } + + return res; + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/incremental/OffheapIncrementalIndexTestSpec.java b/processing/src/test/java/org/apache/druid/segment/incremental/OffheapIncrementalIndexTestSpec.java new file mode 100644 index 000000000000..925973199113 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/incremental/OffheapIncrementalIndexTestSpec.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.incremental; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Supplier; +import org.apache.druid.collections.CloseableStupidPool; +import org.apache.druid.utils.JvmUtils; + +import javax.annotation.Nullable; +import java.io.Closeable; +import java.nio.ByteBuffer; + +/** + * OffheapIncrementalIndexTestSpec describes the off-heap indexing method for data ingestion. + * It also acts as a ByteBuffer supplier for the created off-heap incremental index. + * + * Note: since the off-heap incremental index is not yet supported in production ingestion, we define its spec here + * only for testing purposes. + */ +public class OffheapIncrementalIndexTestSpec implements AppendableIndexSpec, Supplier, Closeable +{ + public static final String TYPE = "offheap"; + static final int DEFAULT_BUFFER_SIZE = 1 << 23; + static final int DEFAULT_CACHE_SIZE = 1 << 30; + + final int bufferSize; + final int cacheSize; + + final CloseableStupidPool bufferPool; + + @JsonCreator + public OffheapIncrementalIndexTestSpec( + final @JsonProperty("bufferSize") @Nullable Integer bufferSize, + final @JsonProperty("cacheSize") @Nullable Integer cacheSize + ) + { + this.bufferSize = bufferSize != null && bufferSize > 0 ? bufferSize : DEFAULT_BUFFER_SIZE; + this.cacheSize = cacheSize != null && cacheSize > this.bufferSize ? cacheSize : DEFAULT_CACHE_SIZE; + this.bufferPool = new CloseableStupidPool<>( + "Off-heap incremental-index buffer pool", + this, + 0, + this.cacheSize / this.bufferSize + ); + } + + @JsonProperty + public int getBufferSize() + { + return bufferSize; + } + + @JsonProperty + public int getCacheSize() + { + return cacheSize; + } + + @Override + public AppendableIndexBuilder builder() + { + return new OffheapIncrementalIndex.Builder().setBufferPool(bufferPool); + } + + @Override + public long getDefaultMaxBytesInMemory() + { + // In the realtime node, the entire JVM's direct memory is utilized for ingestion and persist operations. + // But maxBytesInMemory only refers to the active index size and not to the index being flushed to disk and the + // persist buffer. + // To account for that, we set default to 1/2 of the max JVM's direct memory. + return JvmUtils.getRuntimeInfo().getDirectMemorySizeBytes() / 2; + } + + // Supplier and Closeable interface implementation + + @Override + public ByteBuffer get() + { + return ByteBuffer.allocateDirect(bufferSize); + } + + @Override + public void close() + { + bufferPool.close(); + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java new file mode 100644 index 000000000000..3fde3338861d --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorObjectSelectorTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.virtual; + +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.vector.ExprEvalStringVector; +import org.apache.druid.math.expr.vector.ExprEvalVector; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.easymock.EasyMock; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class ExpressionVectorObjectSelectorTest +{ + private static final int MAX_SIZE = 8; + private Expr.VectorInputBinding binding; + private ExprVectorProcessor vectorProcessor; + private ExpressionVectorObjectSelector expressionVectorValueSelector; + + @Before + public void setUp() + { + binding = EasyMock.createMock(Expr.VectorInputBinding.class); + vectorProcessor = EasyMock.createMock(ExprVectorProcessor.class); + EasyMock.expect(binding.getMaxVectorSize()).andReturn(MAX_SIZE).once(); + EasyMock.replay(binding, vectorProcessor); + expressionVectorValueSelector = new ExpressionVectorObjectSelector(vectorProcessor, binding); + EasyMock.reset(binding, vectorProcessor); + } + + @After + public void tearDown() + { + EasyMock.verify(binding, vectorProcessor); + } + + @Test + public void testSelectObject() + { + final String[] vector = new String[]{"1", "2", null, "3"}; + ExprEvalVector vectorEval = new ExprEvalStringVector(vector); + EasyMock.expect(binding.getCurrentVectorId()).andReturn(1).anyTimes(); + EasyMock.expect(vectorProcessor.evalVector(binding)).andReturn(vectorEval).once(); + EasyMock.replay(binding, vectorProcessor); + + Object[] vector1 = expressionVectorValueSelector.getObjectVector(); + Object[] vector2 = expressionVectorValueSelector.getObjectVector(); + + Assert.assertArrayEquals(vector1, vector2); + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelectorTest.java b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelectorTest.java new file mode 100644 index 000000000000..d7c27b1fb6c0 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/virtual/ExpressionVectorValueSelectorTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.virtual; + +import org.apache.druid.math.expr.Expr; +import org.apache.druid.math.expr.vector.ExprEvalDoubleVector; +import org.apache.druid.math.expr.vector.ExprEvalLongVector; +import org.apache.druid.math.expr.vector.ExprEvalVector; +import org.apache.druid.math.expr.vector.ExprVectorProcessor; +import org.easymock.EasyMock; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class ExpressionVectorValueSelectorTest +{ + private static final int MAX_SIZE = 8; + private Expr.VectorInputBinding binding; + private ExprVectorProcessor vectorProcessor; + private ExpressionVectorValueSelector expressionVectorValueSelector; + + @Before + public void setUp() + { + binding = EasyMock.createMock(Expr.VectorInputBinding.class); + vectorProcessor = EasyMock.createMock(ExprVectorProcessor.class); + EasyMock.expect(binding.getMaxVectorSize()).andReturn(MAX_SIZE).once(); + EasyMock.replay(binding, vectorProcessor); + expressionVectorValueSelector = new ExpressionVectorValueSelector(vectorProcessor, binding); + EasyMock.reset(binding, vectorProcessor); + } + + @After + public void tearDown() + { + EasyMock.verify(binding, vectorProcessor); + } + + @Test + public void testLongVector() + { + final long[] vector = new long[]{1L, 2L, 0L, 3L}; + final boolean[] nulls = new boolean[]{false, false, true, false}; + ExprEvalVector vectorEval = new ExprEvalLongVector(vector, nulls); + EasyMock.expect(binding.getCurrentVectorId()).andReturn(1).anyTimes(); + EasyMock.expect(vectorProcessor.evalVector(binding)).andReturn(vectorEval).once(); + EasyMock.replay(binding, vectorProcessor); + + long[] vector1 = expressionVectorValueSelector.getLongVector(); + boolean[] bools1 = expressionVectorValueSelector.getNullVector(); + long[] vector2 = expressionVectorValueSelector.getLongVector(); + boolean[] bools2 = expressionVectorValueSelector.getNullVector(); + + Assert.assertArrayEquals(vector1, vector2); + Assert.assertArrayEquals(bools1, bools2); + } + + @Test + public void testDoubleVector() + { + final double[] vector = new double[]{1.0, 2.0, 0.0, 3.0}; + final boolean[] nulls = new boolean[]{false, false, true, false}; + ExprEvalVector vectorEval = new ExprEvalDoubleVector(vector, nulls); + EasyMock.expect(binding.getCurrentVectorId()).andReturn(1).anyTimes(); + EasyMock.expect(vectorProcessor.evalVector(binding)).andReturn(vectorEval).once(); + EasyMock.replay(binding, vectorProcessor); + + double[] vector1 = expressionVectorValueSelector.getDoubleVector(); + boolean[] bools1 = expressionVectorValueSelector.getNullVector(); + double[] vector2 = expressionVectorValueSelector.getDoubleVector(); + boolean[] bools2 = expressionVectorValueSelector.getNullVector(); + + Assert.assertArrayEquals(vector1, vector2, 0.0); + Assert.assertArrayEquals(bools1, bools2); + } + + @Test + public void testFloatVector() + { + final double[] vector = new double[]{1.0, 2.0, 0.0, 3.0}; + final boolean[] nulls = new boolean[]{false, false, true, false}; + ExprEvalVector vectorEval = new ExprEvalDoubleVector(vector, nulls); + EasyMock.expect(binding.getCurrentVectorId()).andReturn(1).anyTimes(); + EasyMock.expect(binding.getCurrentVectorSize()).andReturn(4).anyTimes(); + EasyMock.expect(vectorProcessor.evalVector(binding)).andReturn(vectorEval).once(); + EasyMock.replay(binding, vectorProcessor); + + float[] vector1 = expressionVectorValueSelector.getFloatVector(); + boolean[] bools1 = expressionVectorValueSelector.getNullVector(); + float[] vector2 = expressionVectorValueSelector.getFloatVector(); + boolean[] bools2 = expressionVectorValueSelector.getNullVector(); + + for (int i = 0; i < vector1.length; i++) { + Assert.assertEquals(vector1[i], vector2[i], 0.0); + } + Assert.assertArrayEquals(bools1, bools2); + } +} diff --git a/server/src/main/java/org/apache/druid/discovery/BaseNodeRoleWatcher.java b/server/src/main/java/org/apache/druid/discovery/BaseNodeRoleWatcher.java new file mode 100644 index 000000000000..793293a9aaaf --- /dev/null +++ b/server/src/main/java/org/apache/druid/discovery/BaseNodeRoleWatcher.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.discovery; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.errorprone.annotations.concurrent.GuardedBy; +import org.apache.druid.java.util.common.logger.Logger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * Common code used by various implementations of DruidNodeDiscovery. + * + * User code is supposed to arrange for following methods to be called, + * {@link #childAdded(DiscoveryDruidNode)} + * {@link #childRemoved(DiscoveryDruidNode)} + * {@link #cacheInitialized()} + * {@link #resetNodes(Map)} + * + * Then {@link #registerListener(DruidNodeDiscovery.Listener)} and {@link #getAllNodes()} can be delegated to the + * implementation here. + */ +public class BaseNodeRoleWatcher +{ + private static final Logger LOGGER = new Logger(BaseNodeRoleWatcher.class); + + private final NodeRole nodeRole; + + /** + * hostAndPort -> DiscoveryDruidNode + */ + private final ConcurrentMap nodes = new ConcurrentHashMap<>(); + private final Collection unmodifiableNodes = Collections.unmodifiableCollection(nodes.values()); + + private final ExecutorService listenerExecutor; + + private final List nodeListeners = new ArrayList<>(); + + private final Object lock = new Object(); + + private final CountDownLatch cacheInitialized = new CountDownLatch(1); + + public BaseNodeRoleWatcher( + ExecutorService listenerExecutor, + NodeRole nodeRole + ) + { + this.listenerExecutor = listenerExecutor; + this.nodeRole = nodeRole; + } + + public Collection getAllNodes() + { + boolean nodeViewInitialized; + try { + nodeViewInitialized = cacheInitialized.await((long) 30, TimeUnit.SECONDS); + } + catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + nodeViewInitialized = false; + } + if (!nodeViewInitialized) { + LOGGER.info( + "Cache for node role [%s] not initialized yet; getAllNodes() might not return full information.", + nodeRole.getJsonName() + ); + } + return unmodifiableNodes; + } + + public void registerListener(DruidNodeDiscovery.Listener listener) + { + synchronized (lock) { + // No need to wait on CountDownLatch, because we are holding the lock under which it could only be counted down. + if (cacheInitialized.getCount() == 0) { + // It is important to take a snapshot here as list of nodes might change by the time listeners process + // the changes. + List currNodes = Lists.newArrayList(nodes.values()); + safeSchedule( + () -> { + listener.nodesAdded(currNodes); + listener.nodeViewInitialized(); + }, + "Exception occured in nodesAdded([%s]) in listener [%s].", currNodes, listener + ); + } + nodeListeners.add(listener); + } + } + + public void childAdded(DiscoveryDruidNode druidNode) + { + synchronized (lock) { + if (!nodeRole.equals(druidNode.getNodeRole())) { + LOGGER.error( + "Node[%s] of role[%s] addition ignored due to mismatched role (expected role[%s]).", + druidNode.getDruidNode().getUriToUse(), + druidNode.getNodeRole().getJsonName(), + nodeRole.getJsonName() + ); + return; + } + + LOGGER.info("Node[%s] of role[%s] detected.", druidNode.getDruidNode().getUriToUse(), nodeRole.getJsonName()); + + addNode(druidNode); + } + } + + @GuardedBy("lock") + private void addNode(DiscoveryDruidNode druidNode) + { + DiscoveryDruidNode prev = nodes.putIfAbsent(druidNode.getDruidNode().getHostAndPortToUse(), druidNode); + if (prev == null) { + // No need to wait on CountDownLatch, because we are holding the lock under which it could only be counted down. + if (cacheInitialized.getCount() == 0) { + List newNode = ImmutableList.of(druidNode); + for (DruidNodeDiscovery.Listener listener : nodeListeners) { + safeSchedule( + () -> listener.nodesAdded(newNode), + "Exception occured in nodeAdded(node=[%s]) in listener [%s].", + druidNode.getDruidNode().getHostAndPortToUse(), + listener + ); + } + } + } else { + LOGGER.error( + "Node[%s] of role[%s] discovered but existed already [%s].", + druidNode.getDruidNode().getUriToUse(), + nodeRole.getJsonName(), + prev + ); + } + } + + public void childRemoved(DiscoveryDruidNode druidNode) + { + synchronized (lock) { + if (!nodeRole.equals(druidNode.getNodeRole())) { + LOGGER.error( + "Node[%s] of role[%s] removal ignored due to mismatched role (expected role[%s]).", + druidNode.getDruidNode().getUriToUse(), + druidNode.getNodeRole().getJsonName(), + nodeRole.getJsonName() + ); + return; + } + + LOGGER.info("Node[%s] of role[%s] went offline.", druidNode.getDruidNode().getUriToUse(), nodeRole.getJsonName()); + + removeNode(druidNode); + } + } + + @GuardedBy("lock") + private void removeNode(DiscoveryDruidNode druidNode) + { + DiscoveryDruidNode prev = nodes.remove(druidNode.getDruidNode().getHostAndPortToUse()); + + if (prev == null) { + LOGGER.error( + "Noticed disappearance of unknown druid node [%s] of role[%s].", + druidNode.getDruidNode().getUriToUse(), + druidNode.getNodeRole().getJsonName() + ); + return; + } + + // No need to wait on CountDownLatch, because we are holding the lock under which it could only be counted down. + if (cacheInitialized.getCount() == 0) { + List nodeRemoved = ImmutableList.of(druidNode); + for (DruidNodeDiscovery.Listener listener : nodeListeners) { + safeSchedule( + () -> listener.nodesRemoved(nodeRemoved), + "Exception occured in nodeRemoved(node[%s] of role[%s]) in listener [%s].", + druidNode.getDruidNode().getUriToUse(), + druidNode.getNodeRole().getJsonName(), + listener + ); + } + } + } + + public void cacheInitialized() + { + synchronized (lock) { + // No need to wait on CountDownLatch, because we are holding the lock under which it could only be + // counted down. + if (cacheInitialized.getCount() == 0) { + LOGGER.error("cache is already initialized. ignoring cache initialization event."); + return; + } + + LOGGER.info("Node watcher of role[%s] is now initialized.", nodeRole.getJsonName()); + + for (DruidNodeDiscovery.Listener listener : nodeListeners) { + // It is important to take a snapshot here as list of nodes might change by the time listeners process + // the changes. + List currNodes = Lists.newArrayList(nodes.values()); + safeSchedule( + () -> { + listener.nodesAdded(currNodes); + listener.nodeViewInitialized(); + }, + "Exception occured in nodesAdded([%s]) in listener [%s].", + currNodes, + listener + ); + } + + cacheInitialized.countDown(); + } + } + + public void resetNodes(Map fullNodes) + { + synchronized (lock) { + List nodesAdded = new ArrayList<>(); + List nodesDeleted = new ArrayList<>(); + + for (Map.Entry e : fullNodes.entrySet()) { + if (!nodes.containsKey(e.getKey())) { + nodesAdded.add(e.getValue()); + } + } + + for (Map.Entry e : nodes.entrySet()) { + if (!fullNodes.containsKey(e.getKey())) { + nodesDeleted.add(e.getValue()); + } + } + + for (DiscoveryDruidNode node : nodesDeleted) { + nodes.remove(node.getDruidNode().getHostAndPortToUse()); + } + + for (DiscoveryDruidNode node : nodesAdded) { + nodes.put(node.getDruidNode().getHostAndPortToUse(), node); + } + + // No need to wait on CountDownLatch, because we are holding the lock under which it could only be counted down. + if (cacheInitialized.getCount() == 0) { + for (DruidNodeDiscovery.Listener listener : nodeListeners) { + safeSchedule( + () -> { + if (!nodesAdded.isEmpty()) { + listener.nodesAdded(nodesAdded); + } + + if (!nodesDeleted.isEmpty()) { + listener.nodesRemoved(nodesDeleted); + } + }, + "Exception occured in resetNodes in listener [%s].", + listener + ); + } + } + } + } + + private void safeSchedule(Runnable runnable, String errMsgFormat, Object... args) + { + listenerExecutor.submit(() -> { + try { + runnable.run(); + } + catch (Exception ex) { + LOGGER.error(errMsgFormat, args); + } + }); + } +} diff --git a/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java new file mode 100644 index 000000000000..e077aae55f90 --- /dev/null +++ b/server/src/main/java/org/apache/druid/metadata/BasicDataSourceExt.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.metadata; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.dbcp2.ConnectionFactory; +import org.apache.druid.java.util.common.RE; +import org.apache.druid.java.util.common.logger.Logger; + +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +/** + * This class exists so that {@link MetadataStorageConnectorConfig} is asked for password every time a brand new + * connection is established with DB. {@link PasswordProvider} impls such as those based on AWS tokens refresh the + * underlying token periodically since each token is valid for a certain period of time only. + * So, This class overrides (and largely copies due to lack of extensibility), the methods from base class in order to keep + * track of connection properties and call {@link MetadataStorageConnectorConfig#getPassword()} everytime a new + * connection is setup. + */ +public class BasicDataSourceExt extends BasicDataSource +{ + private static final Logger LOGGER = new Logger(BasicDataSourceExt.class); + + private Properties connectionProperties; + private final MetadataStorageConnectorConfig connectorConfig; + + public BasicDataSourceExt(MetadataStorageConnectorConfig connectorConfig) + { + this.connectorConfig = connectorConfig; + this.connectionProperties = new Properties(); + } + + @Override + public void addConnectionProperty(String name, String value) + { + connectionProperties.put(name, value); + super.addConnectionProperty(name, value); + } + + @Override + public void removeConnectionProperty(String name) + { + connectionProperties.remove(name); + super.removeConnectionProperty(name); + } + + @Override + public void setConnectionProperties(String connectionProperties) + { + if (connectionProperties == null) { + throw new NullPointerException("connectionProperties is null"); + } + + String[] entries = connectionProperties.split(";"); + Properties properties = new Properties(); + for (String entry : entries) { + if (entry.length() > 0) { + int index = entry.indexOf('='); + if (index > 0) { + String name = entry.substring(0, index); + String value = entry.substring(index + 1); + properties.setProperty(name, value); + } else { + // no value is empty string which is how java.util.Properties works + properties.setProperty(entry, ""); + } + } + } + this.connectionProperties = properties; + super.setConnectionProperties(connectionProperties); + } + + @VisibleForTesting + public Properties getConnectionProperties() + { + return connectionProperties; + } + + @Override + protected ConnectionFactory createConnectionFactory() throws SQLException + { + Driver driverToUse = getDriver(); + + if (driverToUse == null) { + Class driverFromCCL = null; + if (getDriverClassName() != null) { + try { + try { + if (getDriverClassLoader() == null) { + driverFromCCL = Class.forName(getDriverClassName()); + } else { + driverFromCCL = Class.forName( + getDriverClassName(), true, getDriverClassLoader()); + } + } + catch (ClassNotFoundException cnfe) { + driverFromCCL = Thread.currentThread( + ).getContextClassLoader().loadClass( + getDriverClassName()); + } + } + catch (Exception t) { + String message = "Cannot load JDBC driver class '" + + getDriverClassName() + "'"; + LOGGER.error(t, message); + throw new SQLException(message, t); + } + } + + try { + if (driverFromCCL == null) { + driverToUse = DriverManager.getDriver(getUrl()); + } else { + // Usage of DriverManager is not possible, as it does not + // respect the ContextClassLoader + // N.B. This cast may cause ClassCastException which is handled below + driverToUse = (Driver) driverFromCCL.newInstance(); + if (!driverToUse.acceptsURL(getUrl())) { + throw new SQLException("No suitable driver", "08001"); + } + } + } + catch (Exception t) { + String message = "Cannot create JDBC driver of class '" + + (getDriverClassName() != null ? getDriverClassName() : "") + + "' for connect URL '" + getUrl() + "'"; + LOGGER.error(t, message); + throw new SQLException(message, t); + } + } + + if (driverToUse == null) { + throw new RE("Failed to find the DB Driver"); + } + + final Driver finalDriverToUse = driverToUse; + + return () -> { + String user = connectorConfig.getUser(); + if (user != null) { + connectionProperties.put("user", user); + } else { + log("DBCP DataSource configured without a 'username'"); + } + + // Note: This is the main point of this class where we are getting fresh password before setting up + // every new connection. + String password = connectorConfig.getPassword(); + if (password != null) { + connectionProperties.put("password", password); + } else { + log("DBCP DataSource configured without a 'password'"); + } + + return finalDriverToUse.connect(connectorConfig.getConnectURI(), connectionProperties); + }; + } +} diff --git a/server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java b/server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java new file mode 100644 index 000000000000..ff860eef2e62 --- /dev/null +++ b/server/src/test/java/org/apache/druid/client/indexing/HttpIndexingServiceClientTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.client.indexing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.discovery.DruidLeaderClient; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.http.client.Request; +import org.apache.druid.java.util.http.client.response.StringFullResponseHolder; +import org.easymock.EasyMock; +import org.jboss.netty.buffer.BigEndianHeapChannelBuffer; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponse; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.net.URL; +import java.nio.charset.StandardCharsets; + +public class HttpIndexingServiceClientTest +{ + private HttpIndexingServiceClient httpIndexingServiceClient; + private ObjectMapper jsonMapper; + private DruidLeaderClient druidLeaderClient; + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Before + public void setup() + { + jsonMapper = new DefaultObjectMapper(); + druidLeaderClient = EasyMock.createMock(DruidLeaderClient.class); + httpIndexingServiceClient = new HttpIndexingServiceClient( + jsonMapper, + druidLeaderClient + ); + } + + @Test + public void testSample() throws Exception + { + final SamplerResponse samplerResponse = new SamplerResponse( + 2, + 2, + ImmutableList.of( + new SamplerResponse.SamplerResponseRow( + ImmutableMap.of("time", "2020-01-01", "x", "123", "y", "456"), + ImmutableMap.of("time", "2020-01-01", "x", "123", "y", "456"), + false, + null + ) + ) + ); + + final SamplerSpec samplerSpec = new SamplerSpec() + { + @Override + public SamplerResponse sample() + { + return samplerResponse; + } + }; + + HttpResponse response = EasyMock.createMock(HttpResponse.class); + EasyMock.expect(response.getContent()).andReturn(new BigEndianHeapChannelBuffer(0)); + EasyMock.replay(response); + + StringFullResponseHolder responseHolder = new StringFullResponseHolder( + HttpResponseStatus.OK, + response, + StandardCharsets.UTF_8 + ).addChunk(jsonMapper.writeValueAsString(samplerResponse)); + + EasyMock.expect(druidLeaderClient.go(EasyMock.anyObject(Request.class))) + .andReturn(responseHolder) + .anyTimes(); + + EasyMock.expect(druidLeaderClient.makeRequest(HttpMethod.POST, "/druid/indexer/v1/sampler")) + .andReturn(new Request(HttpMethod.POST, new URL("http://localhost:8090/druid/indexer/v1/sampler"))) + .anyTimes(); + EasyMock.replay(druidLeaderClient); + + final SamplerResponse actualResponse = httpIndexingServiceClient.sample(samplerSpec); + Assert.assertEquals(samplerResponse, actualResponse); + + EasyMock.verify(druidLeaderClient, response); + } + + @Test + public void testSampleError() throws Exception + { + expectedException.expect(RuntimeException.class); + expectedException.expectMessage("Failed to sample with sampler spec"); + expectedException.expectMessage("Please check overlord log"); + + final SamplerResponse samplerResponse = new SamplerResponse( + 2, + 2, + ImmutableList.of( + new SamplerResponse.SamplerResponseRow( + ImmutableMap.of("time", "2020-01-01", "x", "123", "y", "456"), + ImmutableMap.of("time", "2020-01-01", "x", "123", "y", "456"), + false, + null + ) + ) + ); + + final SamplerSpec samplerSpec = new SamplerSpec() + { + @Override + public SamplerResponse sample() + { + return samplerResponse; + } + }; + HttpResponse response = EasyMock.createMock(HttpResponse.class); + EasyMock.expect(response.getContent()).andReturn(new BigEndianHeapChannelBuffer(0)); + EasyMock.replay(response); + + StringFullResponseHolder responseHolder = new StringFullResponseHolder( + HttpResponseStatus.INTERNAL_SERVER_ERROR, + response, + StandardCharsets.UTF_8 + ).addChunk(""); + + EasyMock.expect(druidLeaderClient.go(EasyMock.anyObject(Request.class))) + .andReturn(responseHolder) + .anyTimes(); + + EasyMock.expect(druidLeaderClient.makeRequest(HttpMethod.POST, "/druid/indexer/v1/sampler")) + .andReturn(new Request(HttpMethod.POST, new URL("http://localhost:8090/druid/indexer/v1/sampler"))) + .anyTimes(); + EasyMock.replay(druidLeaderClient); + + + httpIndexingServiceClient.sample(samplerSpec); + EasyMock.verify(druidLeaderClient, response); + } +} diff --git a/server/src/test/java/org/apache/druid/discovery/BaseNodeRoleWatcherTest.java b/server/src/test/java/org/apache/druid/discovery/BaseNodeRoleWatcherTest.java new file mode 100644 index 000000000000..1bf5ba25040b --- /dev/null +++ b/server/src/test/java/org/apache/druid/discovery/BaseNodeRoleWatcherTest.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.discovery; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.server.DruidNode; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +public class BaseNodeRoleWatcherTest +{ + @Test(timeout = 60_000L) + public void testGeneralUseSimulation() + { + BaseNodeRoleWatcher nodeRoleWatcher = new BaseNodeRoleWatcher( + Execs.directExecutor(), + NodeRole.BROKER + ); + + DiscoveryDruidNode broker1 = buildDiscoveryDruidNode(NodeRole.BROKER, "broker1"); + DiscoveryDruidNode broker2 = buildDiscoveryDruidNode(NodeRole.BROKER, "broker2"); + DiscoveryDruidNode broker3 = buildDiscoveryDruidNode(NodeRole.BROKER, "broker3"); + + DiscoveryDruidNode notBroker = new DiscoveryDruidNode( + new DruidNode("s3", "h3", false, 8080, null, true, false), + NodeRole.COORDINATOR, + ImmutableMap.of() + ); + + TestListener listener1 = new TestListener(); + TestListener listener2 = new TestListener(); + TestListener listener3 = new TestListener(); + + nodeRoleWatcher.registerListener(listener1); + nodeRoleWatcher.childAdded(broker1); + nodeRoleWatcher.childAdded(broker2); + nodeRoleWatcher.childAdded(notBroker); + nodeRoleWatcher.childAdded(broker3); + nodeRoleWatcher.registerListener(listener2); + nodeRoleWatcher.childRemoved(broker2); + + assertListener(listener1, false, Collections.emptyList(), Collections.emptyList()); + assertListener(listener2, false, Collections.emptyList(), Collections.emptyList()); + + nodeRoleWatcher.cacheInitialized(); + + nodeRoleWatcher.registerListener(listener3); + + List presentNodes = new ArrayList<>(nodeRoleWatcher.getAllNodes()); + Assert.assertEquals(2, presentNodes.size()); + Assert.assertTrue(presentNodes.contains(broker1)); + Assert.assertTrue(presentNodes.contains(broker3)); + + assertListener(listener1, true, presentNodes, Collections.emptyList()); + assertListener(listener2, true, presentNodes, Collections.emptyList()); + assertListener(listener3, true, presentNodes, Collections.emptyList()); + + nodeRoleWatcher.childRemoved(notBroker); + nodeRoleWatcher.childRemoved(broker2); + nodeRoleWatcher.childAdded(broker2); + nodeRoleWatcher.childRemoved(broker3); + nodeRoleWatcher.childAdded(broker1); + + Assert.assertEquals(ImmutableSet.of(broker2, broker1), new HashSet<>(nodeRoleWatcher.getAllNodes())); + + List nodesAdded = new ArrayList<>(presentNodes); + nodesAdded.add(broker2); + + List nodesRemoved = new ArrayList<>(); + nodesRemoved.add(broker3); + + assertListener(listener1, true, nodesAdded, nodesRemoved); + assertListener(listener2, true, nodesAdded, nodesRemoved); + assertListener(listener3, true, nodesAdded, nodesRemoved); + + LinkedHashMap resetNodes = new LinkedHashMap<>(); + resetNodes.put(broker2.getDruidNode().getHostAndPortToUse(), broker2); + resetNodes.put(broker3.getDruidNode().getHostAndPortToUse(), broker3); + + nodeRoleWatcher.resetNodes(resetNodes); + + Assert.assertEquals(ImmutableSet.of(broker2, broker3), new HashSet<>(nodeRoleWatcher.getAllNodes())); + + nodesAdded.add(broker3); + nodesRemoved.add(broker1); + + assertListener(listener1, true, nodesAdded, nodesRemoved); + assertListener(listener2, true, nodesAdded, nodesRemoved); + assertListener(listener3, true, nodesAdded, nodesRemoved); + } + + private DiscoveryDruidNode buildDiscoveryDruidNode(NodeRole role, String host) + { + return new DiscoveryDruidNode( + new DruidNode("s", host, false, 8080, null, true, false), + role, + ImmutableMap.of() + ); + } + + private void assertListener(TestListener listener, boolean nodeViewInitialized, List nodesAdded, List nodesRemoved) + { + Assert.assertEquals(nodeViewInitialized, listener.nodeViewInitialized.get()); + Assert.assertEquals(nodesAdded, listener.nodesAddedList); + Assert.assertEquals(nodesRemoved, listener.nodesRemovedList); + } + + public static class TestListener implements DruidNodeDiscovery.Listener + { + private final AtomicBoolean nodeViewInitialized = new AtomicBoolean(false); + private final List nodesAddedList = new ArrayList<>(); + private final List nodesRemovedList = new ArrayList<>(); + + @Override + public void nodesAdded(Collection nodes) + { + nodesAddedList.addAll(nodes); + } + + @Override + public void nodesRemoved(Collection nodes) + { + nodesRemovedList.addAll(nodes); + } + + @Override + public void nodeViewInitialized() + { + if (!nodeViewInitialized.compareAndSet(false, true)) { + throw new RuntimeException("NodeViewInitialized called again!"); + } + } + } +} diff --git a/server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java b/server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java new file mode 100644 index 000000000000..1f8af8f6c981 --- /dev/null +++ b/server/src/test/java/org/apache/druid/metadata/BasicDataSourceExtTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.metadata; + +import org.apache.commons.dbcp2.ConnectionFactory; +import org.assertj.core.util.Lists; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Test; + +import java.sql.Driver; +import java.util.List; +import java.util.Properties; + +public class BasicDataSourceExtTest +{ + @Test + public void testCreateConnectionFactory() throws Exception + { + MetadataStorageConnectorConfig connectorConfig = new MetadataStorageConnectorConfig() + { + private final List passwords = Lists.newArrayList("pwd1", "pwd2"); + + @Override + public String getUser() + { + return "testuser"; + } + + @Override + public String getPassword() + { + return passwords.remove(0); + } + }; + + BasicDataSourceExt basicDataSourceExt = new BasicDataSourceExt(connectorConfig); + + basicDataSourceExt.setConnectionProperties("p1=v1"); + basicDataSourceExt.addConnectionProperty("p2", "v2"); + + Driver driver = EasyMock.mock(Driver.class); + Capture uriArg = Capture.newInstance(); + Capture propsArg = Capture.newInstance(); + EasyMock.expect(driver.connect(EasyMock.capture(uriArg), EasyMock.capture(propsArg))).andReturn(null).times(2); + EasyMock.replay(driver); + + basicDataSourceExt.setDriver(driver); + + ConnectionFactory connectionFactory = basicDataSourceExt.createConnectionFactory(); + + Properties expectedProps = new Properties(); + expectedProps.put("p1", "v1"); + expectedProps.put("p2", "v2"); + expectedProps.put("user", connectorConfig.getUser()); + + + Assert.assertNull(connectionFactory.createConnection()); + Assert.assertEquals(connectorConfig.getConnectURI(), uriArg.getValue()); + + expectedProps.put("password", "pwd1"); + Assert.assertEquals(expectedProps, propsArg.getValue()); + + Assert.assertNull(connectionFactory.createConnection()); + Assert.assertEquals(connectorConfig.getConnectURI(), uriArg.getValue()); + + expectedProps.put("password", "pwd2"); + Assert.assertEquals(expectedProps, propsArg.getValue()); + } + + @Test + public void testConnectionPropertiesHanding() + { + BasicDataSourceExt basicDataSourceExt = new BasicDataSourceExt(EasyMock.mock(MetadataStorageConnectorConfig.class)); + Properties expectedProps = new Properties(); + + basicDataSourceExt.setConnectionProperties(""); + Assert.assertEquals(expectedProps, basicDataSourceExt.getConnectionProperties()); + + basicDataSourceExt.setConnectionProperties("p0;p1=v1;p2=v2;p3=v3"); + basicDataSourceExt.addConnectionProperty("p4", "v4"); + basicDataSourceExt.addConnectionProperty("p5", "v5"); + basicDataSourceExt.removeConnectionProperty("p2"); + basicDataSourceExt.removeConnectionProperty("p5"); + + expectedProps.put("p0", ""); + expectedProps.put("p1", "v1"); + expectedProps.put("p3", "v3"); + expectedProps.put("p4", "v4"); + + Assert.assertEquals(expectedProps, basicDataSourceExt.getConnectionProperties()); + + + } +} diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java new file mode 100644 index 000000000000..f7c85b2ccd08 --- /dev/null +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.realtime.appenderator; + +import com.google.common.collect.ImmutableMap; +import org.apache.druid.client.cache.CacheConfig; +import org.apache.druid.client.cache.CachePopulatorStats; +import org.apache.druid.client.cache.MapCache; +import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.indexing.worker.config.WorkerConfig; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; +import org.apache.druid.query.Druids; +import org.apache.druid.query.scan.ScanQuery; +import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; +import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.incremental.NoopRowIngestionMeters; +import org.apache.druid.segment.incremental.ParseExceptionHandler; +import org.apache.druid.segment.indexing.DataSchema; +import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; +import org.apache.druid.segment.join.NoopJoinableFactory; +import org.apache.druid.segment.loading.NoopDataSegmentPusher; +import org.apache.druid.segment.realtime.FireDepartmentMetrics; +import org.apache.druid.segment.writeout.OnHeapMemorySegmentWriteOutMediumFactory; +import org.apache.druid.server.metrics.NoopServiceEmitter; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.util.Collections; + +public class UnifiedIndexerAppenderatorsManagerTest +{ + @Rule + public final ExpectedException expectedException = ExpectedException.none(); + + private final UnifiedIndexerAppenderatorsManager manager = new UnifiedIndexerAppenderatorsManager( + Execs.directExecutor(), + NoopJoinableFactory.INSTANCE, + new WorkerConfig(), + MapCache.create(10), + new CacheConfig(), + new CachePopulatorStats(), + TestHelper.makeJsonMapper(), + new NoopServiceEmitter(), + () -> new DefaultQueryRunnerFactoryConglomerate(ImmutableMap.of()) + ); + + private final Appenderator appenderator = manager.createOfflineAppenderatorForTask( + "taskId", + new DataSchema( + "myDataSource", + new TimestampSpec("__time", "millis", null), + null, + null, + new UniformGranularitySpec(Granularities.HOUR, Granularities.HOUR, false, Collections.emptyList()), + null + ), + EasyMock.createMock(AppenderatorConfig.class), + new FireDepartmentMetrics(), + new NoopDataSegmentPusher(), + TestHelper.makeJsonMapper(), + TestHelper.getTestIndexIO(), + TestHelper.getTestIndexMergerV9(OnHeapMemorySegmentWriteOutMediumFactory.instance()), + new NoopRowIngestionMeters(), + new ParseExceptionHandler(new NoopRowIngestionMeters(), false, 0, 0) + ); + + @Test + public void test_getBundle_knownDataSource() + { + final UnifiedIndexerAppenderatorsManager.DatasourceBundle bundle = manager.getBundle( + Druids.newScanQueryBuilder() + .dataSource(appenderator.getDataSource()) + .intervals(new MultipleIntervalSegmentSpec(Intervals.ONLY_ETERNITY)) + .build() + ); + + Assert.assertEquals("myDataSource", bundle.getWalker().getDataSource()); + } + + @Test + public void test_getBundle_unknownDataSource() + { + final ScanQuery query = Druids.newScanQueryBuilder() + .dataSource("unknown") + .intervals(new MultipleIntervalSegmentSpec(Intervals.ONLY_ETERNITY)) + .build(); + + expectedException.expect(IllegalArgumentException.class); + expectedException.expectMessage("Could not find segment walker for datasource"); + + manager.getBundle(query); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/SqlPlanningException.java b/sql/src/main/java/org/apache/druid/sql/SqlPlanningException.java new file mode 100644 index 000000000000..98f3ed319dad --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/SqlPlanningException.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.calcite.sql.parser.SqlParseException; +import org.apache.calcite.tools.ValidationException; +import org.apache.druid.query.BadQueryException; + +/** + * An exception for SQL query planning failures. + */ +public class SqlPlanningException extends BadQueryException +{ + public enum PlanningError + { + SQL_PARSE_ERROR("SQL parse failed", SqlParseException.class.getName()), + VALIDATION_ERROR("Plan validation failed", ValidationException.class.getName()); + + private final String errorCode; + private final String errorClass; + + PlanningError(String errorCode, String errorClass) + { + this.errorCode = errorCode; + this.errorClass = errorClass; + } + + public String getErrorCode() + { + return errorCode; + } + + public String getErrorClass() + { + return errorClass; + } + } + + public SqlPlanningException(SqlParseException e) + { + this(PlanningError.SQL_PARSE_ERROR, e.getMessage()); + } + + public SqlPlanningException(ValidationException e) + { + this(PlanningError.VALIDATION_ERROR, e.getMessage()); + } + + private SqlPlanningException(PlanningError planningError, String errorMessage) + { + this(planningError.errorCode, errorMessage, planningError.errorClass); + } + + @JsonCreator + private SqlPlanningException( + @JsonProperty("error") String errorCode, + @JsonProperty("errorMessage") String errorMessage, + @JsonProperty("errorClass") String errorClass + ) + { + super(errorCode, errorMessage, errorClass); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/GroupingSqlAggregator.java b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/GroupingSqlAggregator.java new file mode 100644 index 000000000000..bbff23f2375e --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/aggregation/builtin/GroupingSqlAggregator.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.aggregation.builtin; + +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.GroupingAggregatorFactory; +import org.apache.druid.segment.VirtualColumn; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.sql.calcite.aggregation.Aggregation; +import org.apache.druid.sql.calcite.aggregation.SqlAggregator; +import org.apache.druid.sql.calcite.expression.DruidExpression; +import org.apache.druid.sql.calcite.expression.Expressions; +import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.sql.calcite.rel.VirtualColumnRegistry; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class GroupingSqlAggregator implements SqlAggregator +{ + @Override + public SqlAggFunction calciteFunction() + { + return SqlStdOperatorTable.GROUPING; + } + + @Nullable + @Override + public Aggregation toDruidAggregation( + PlannerContext plannerContext, + RowSignature rowSignature, + VirtualColumnRegistry virtualColumnRegistry, + RexBuilder rexBuilder, + String name, + AggregateCall aggregateCall, + Project project, + List existingAggregations, + boolean finalizeAggregations + ) + { + List arguments = aggregateCall.getArgList() + .stream() + .map(i -> getColumnName( + plannerContext, + rowSignature, + project, + virtualColumnRegistry, + i + )) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + if (arguments.size() < aggregateCall.getArgList().size()) { + return null; + } + + for (Aggregation existing : existingAggregations) { + for (AggregatorFactory factory : existing.getAggregatorFactories()) { + if (!(factory instanceof GroupingAggregatorFactory)) { + continue; + } + GroupingAggregatorFactory groupingFactory = (GroupingAggregatorFactory) factory; + if (groupingFactory.getGroupings().equals(arguments) + && groupingFactory.getName().equals(name)) { + return Aggregation.create(groupingFactory); + } + } + } + AggregatorFactory factory = new GroupingAggregatorFactory(name, arguments); + return Aggregation.create(factory); + } + + @Nullable + private String getColumnName( + PlannerContext plannerContext, + RowSignature rowSignature, + Project project, + VirtualColumnRegistry virtualColumnRegistry, + int fieldNumber + ) + { + RexNode node = Expressions.fromFieldAccess(rowSignature, project, fieldNumber); + if (null == node) { + return null; + } + DruidExpression expression = Expressions.toDruidExpression(plannerContext, rowSignature, node); + if (null == expression) { + return null; + } + if (expression.isDirectColumnAccess()) { + return expression.getDirectColumn(); + } + + VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( + plannerContext, + expression, + node.getType() + ); + return virtualColumn.getOutputName(); + } +} diff --git a/web-console/src/bootstrap/ace.ts b/web-console/src/bootstrap/ace.ts new file mode 100644 index 000000000000..f3689a145162 --- /dev/null +++ b/web-console/src/bootstrap/ace.ts @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import 'brace'; // Import Ace editor and all the sub components used in the app +import 'brace/ext/language_tools'; +import 'brace/theme/solarized_dark'; + +import '../ace-modes/dsql'; +import '../ace-modes/hjson'; diff --git a/web-console/src/components/numeric-input-with-default/__snapshots__/numeric-input-with-default.spec.tsx.snap b/web-console/src/components/numeric-input-with-default/__snapshots__/numeric-input-with-default.spec.tsx.snap new file mode 100644 index 000000000000..75335f79dcb5 --- /dev/null +++ b/web-console/src/components/numeric-input-with-default/__snapshots__/numeric-input-with-default.spec.tsx.snap @@ -0,0 +1,19 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`NumericInputWithDefault matches snapshot 1`] = ` + +`; diff --git a/web-console/src/components/numeric-input-with-default/numeric-input-with-default.spec.tsx b/web-console/src/components/numeric-input-with-default/numeric-input-with-default.spec.tsx new file mode 100644 index 000000000000..b0d2b61d2ce9 --- /dev/null +++ b/web-console/src/components/numeric-input-with-default/numeric-input-with-default.spec.tsx @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { shallow } from 'enzyme'; +import React from 'react'; + +import { NumericInputWithDefault } from './numeric-input-with-default'; + +describe('NumericInputWithDefault', () => { + it('matches snapshot', () => { + const numericInputWithDefault = shallow(); + + expect(numericInputWithDefault).toMatchSnapshot(); + }); +}); diff --git a/web-console/src/components/numeric-input-with-default/numeric-input-with-default.tsx b/web-console/src/components/numeric-input-with-default/numeric-input-with-default.tsx new file mode 100644 index 000000000000..fb731ac6f7db --- /dev/null +++ b/web-console/src/components/numeric-input-with-default/numeric-input-with-default.tsx @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { HTMLInputProps, INumericInputProps, NumericInput } from '@blueprintjs/core'; +import React, { useState } from 'react'; + +export type NumericInputWithDefaultProps = HTMLInputProps & INumericInputProps; + +export const NumericInputWithDefault = React.memo(function NumericInputWithDefault( + props: NumericInputWithDefaultProps, +) { + const { value, defaultValue, onValueChange, onBlur, ...rest } = props; + const [hasChanged, setHasChanged] = useState(false); + + let effectiveValue = value; + if (effectiveValue == null) { + effectiveValue = hasChanged ? '' : typeof defaultValue !== 'undefined' ? defaultValue : ''; + } + + return ( + { + setHasChanged(true); + if (!onValueChange) return; + return onValueChange(valueAsNumber, valueAsString, inputElement); + }} + onBlur={e => { + setHasChanged(false); + if (!onBlur) return; + return onBlur(e); + }} + {...rest} + /> + ); +}); diff --git a/web-console/src/singletons/api.spec.ts b/web-console/src/singletons/api.spec.ts new file mode 100644 index 000000000000..4ff69f32157e --- /dev/null +++ b/web-console/src/singletons/api.spec.ts @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Api } from './api'; + +describe('Api', () => { + it('escapes stuff', () => { + expect(Api.encodePath('wikipedia')).toEqual('wikipedia'); + expect(Api.encodePath(`wi%ki?pe#dia&'[]`)).toEqual('wi%25ki%3Fpe%23dia%26%27%5B%5D'); + }); + + describe(`with BigInt`, () => { + it('works as expected', () => { + const res = (Api.getDefaultConfig().transformResponse as any)[0](`{"x":9223372036854775799}`); + expect(typeof res).toEqual('object'); + expect(typeof res.x).toEqual('bigint'); + expect(String(res.x)).toEqual('9223372036854775799'); + }); + }); + + describe(`without BigInt`, () => { + const originalBigInt = (global as any).BigInt; + + beforeAll(() => { + (global as any).BigInt = null; + }); + + afterAll(() => { + (global as any).BigInt = originalBigInt; + }); + + it('works as expected', () => { + const res = (Api.getDefaultConfig().transformResponse as any)[0](`{"x":9223372036854775799}`); + expect(typeof res).toEqual('object'); + expect(typeof res.x).toEqual('number'); + expect(String(res.x)).toEqual('9223372036854776000'); + }); + }); +}); diff --git a/web-console/src/singletons/api.ts b/web-console/src/singletons/api.ts new file mode 100644 index 000000000000..7a05bdd398c8 --- /dev/null +++ b/web-console/src/singletons/api.ts @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import axios, { AxiosInstance, AxiosRequestConfig } from 'axios'; +import * as JSONBig from 'json-bigint-native'; + +export class Api { + static instance: AxiosInstance; + + static initialize(config?: AxiosRequestConfig): void { + Api.instance = axios.create(config); + } + + static getDefaultConfig(): AxiosRequestConfig { + return { + headers: {}, + + transformResponse: [ + data => { + if (typeof data === 'string') { + try { + data = JSONBig.parse(data); + } catch (e) { + /* Ignore */ + } + } + return data; + }, + ], + }; + } + + static encodePath(path: string): string { + return path.replace( + /[?#%&'\[\]]/g, + c => + '%' + + c + .charCodeAt(0) + .toString(16) + .toUpperCase(), + ); + } +} diff --git a/web-console/src/singletons/index.ts b/web-console/src/singletons/index.ts new file mode 100644 index 000000000000..bf21dfcfe8a5 --- /dev/null +++ b/web-console/src/singletons/index.ts @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './api'; +export * from './toaster'; +export * from './url-baser'; From c0b3ff258020bd17c1e370ac2b71b872b27f25a8 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 15 Jan 2021 18:57:51 +0800 Subject: [PATCH 16/47] Extract the autoScale logic out of SeekableStreamSupervisor to minimize putting more stuff inside there && Make autoscaling algorithm configurable and scalable. --- .../MaterializedViewSupervisor.java | 18 ++ .../MaterializedViewSupervisorSpec.java | 7 + .../kafka/supervisor/KafkaSupervisor.java | 2 +- .../kinesis/supervisor/KinesisSupervisor.java | 2 +- .../supervisor/SupervisorManager.java | 25 ++ .../supervisor/SeekableStreamSupervisor.java | 217 +++------------ .../SeekableStreamSupervisorSpec.java | 24 ++ .../OverlordSecurityResourceFilterTest.java | 8 + .../supervisor/SupervisorManagerTest.java | 7 + .../supervisor/SupervisorResourceTest.java | 7 + .../SeekableStreamSupervisorStateTest.java | 4 +- .../supervisor/NoopSupervisorSpec.java | 26 ++ .../overlord/supervisor/Supervisor.java | 16 ++ .../overlord/supervisor/SupervisorSpec.java | 3 + .../autoscaler/DefaultAutoScaler.java | 254 ++++++++++++++++++ .../autoscaler/DummyAutoScaler.java | 47 ++++ .../autoscaler/SupervisorTaskAutoscaler.java | 27 ++ .../SQLMetadataSupervisorManagerTest.java | 8 + .../druid/metadata/TestSupervisorSpec.java | 8 + 19 files changed, 522 insertions(+), 188 deletions(-) create mode 100644 server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DefaultAutoScaler.java create mode 100644 server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DummyAutoScaler.java create mode 100644 server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/SupervisorTaskAutoscaler.java diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java index 41647d525a59..be6289d2cb00 100644 --- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java +++ b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java @@ -62,6 +62,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; +import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; public class MaterializedViewSupervisor implements Supervisor @@ -282,6 +283,23 @@ public void checkpoint(int taskGroupId, DataSourceMetadata checkpointMetadata) // do nothing } + @Override + public void collectLag(ArrayList lags) + { + } + + @Override + public Runnable buildDynamicAllocationTask(Callable scaleAction) + { + return null; + } + + @Override + public Map getSupervisorTaskInfos() + { + return null; + } + /** * Find intervals in which derived dataSource should rebuild the segments. * Choose the latest intervals to create new HadoopIndexTask and submit it. diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java index db63a7316d1f..c46a22098b7e 100644 --- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java +++ b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java @@ -39,6 +39,7 @@ import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; @@ -361,6 +362,12 @@ public Supervisor createSupervisor() ); } + @Override + public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + { + return null; + } + @Override public List getDataSources() { diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java index ba06e3faaf4d..26e2acaa5296 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java @@ -332,7 +332,7 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() } @Override - protected void collectLag(ArrayList lags) + public void collectLag(ArrayList lags) { Map partitionRecordLag = getPartitionRecordLag(); if (partitionRecordLag == null) { diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java index f8605e431cba..14fa97ffe451 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java @@ -380,7 +380,7 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() // not yet supported, will be implemented in the future maybe? need to find a proper way to measure kinesis lag. @Override - protected void collectLag(ArrayList lags) + public void collectLag(ArrayList lags) { } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java index 5e808e38936f..bf678193d218 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java @@ -23,6 +23,7 @@ import com.google.common.base.Preconditions; import com.google.inject.Inject; import org.apache.druid.indexing.overlord.DataSourceMetadata; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.lifecycle.LifecycleStart; import org.apache.druid.java.util.common.lifecycle.LifecycleStop; @@ -45,6 +46,8 @@ public class SupervisorManager private final MetadataSupervisorManager metadataSupervisorManager; private final ConcurrentHashMap> supervisors = new ConcurrentHashMap<>(); + // SupervisorTaskAutoscaler could be null + private final ConcurrentHashMap autoscalers = new ConcurrentHashMap<>(); private final Object lock = new Object(); private volatile boolean started = false; @@ -146,12 +149,17 @@ public void stop() for (String id : supervisors.keySet()) { try { supervisors.get(id).lhs.stop(false); + SupervisorTaskAutoscaler autoscaler = autoscalers.get(id); + if (autoscaler != null) { + autoscaler.stop(); + } } catch (Exception e) { log.warn(e, "Caught exception while stopping supervisor [%s]", id); } } supervisors.clear(); + autoscalers.clear(); started = false; } @@ -193,6 +201,10 @@ public boolean resetSupervisor(String id, @Nullable DataSourceMetadata dataSourc } supervisor.lhs.reset(dataSourceMetadata); + SupervisorTaskAutoscaler autoscaler = autoscalers.get(id); + if (autoscaler != null) { + autoscaler.reset(); + } return true; } @@ -244,6 +256,12 @@ private boolean possiblyStopAndRemoveSupervisorInternal(String id, boolean write pair.lhs.stop(true); supervisors.remove(id); + SupervisorTaskAutoscaler autoscler = autoscalers.get(id); + if (autoscler != null) { + autoscler.stop(); + autoscalers.remove(id); + } + return true; } @@ -288,9 +306,15 @@ private boolean createAndStartSupervisorInternal(SupervisorSpec spec, boolean pe } Supervisor supervisor; + SupervisorTaskAutoscaler autoscaler; try { supervisor = spec.createSupervisor(); + autoscaler = spec.createAutoscaler(supervisor); + supervisor.start(); + if (autoscaler != null) { + autoscaler.start(); + } } catch (Exception e) { // Supervisor creation or start failed write tombstone only when trying to start a new supervisor @@ -301,6 +325,7 @@ private boolean createAndStartSupervisorInternal(SupervisorSpec spec, boolean pe } supervisors.put(id, Pair.of(supervisor, spec)); + autoscalers.put(id, autoscaler); return true; } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 54904e8b4a85..712627a03c3a 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -40,7 +40,6 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.collections4.queue.CircularFifoQueue; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.indexer.TaskLocation; import org.apache.druid.indexer.TaskState; @@ -106,6 +105,7 @@ import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; @@ -148,7 +148,6 @@ public abstract class SeekableStreamSupervisor scaleAction; + + DynamicAllocationTasksNotice(Callable scaleAction) + { + this.scaleAction = scaleAction; + } + /** * This method will do lags points collection and check dynamic scale action is necessary or not. */ @Override public void handle() { - lock.lock(); try { long nowTime = System.currentTimeMillis(); // Only queue is full and over minTriggerDynamicFrequency can trigger scale out/in @@ -351,98 +356,44 @@ public void handle() log.info("NowTime - dynamicTriggerLastRunTime is [%s]. Defined minTriggerDynamicFrequency is [%s] for dataSource [%s], CLAM DOWN NOW !", nowTime - dynamicTriggerLastRunTime, minTriggerDynamicFrequency, dataSource); return; } - if (!lagMetricsQueue.isAtFullCapacity()) { - log.info("Metrics collection is not at full capacity, may cause unnecessary scale. Skip to check dynamic allocate task : [%s] vs [%s]", lagMetricsQueue.size(), lagMetricsQueue.maxSize()); - return; - } - List lags = collectTotalLags(); - boolean allocationSuccess = dynamicAllocate(lags); + + Integer desriedTaskCount = scaleAction.call(); + boolean allocationSuccess = dynamicAllocate(desriedTaskCount); + if (allocationSuccess) { dynamicTriggerLastRunTime = nowTime; - lagMetricsQueue.clear(); } } - catch (Exception e) { - log.warn(e, "Error, when parse DynamicAllocationTasksNotice"); - } - finally { - lock.unlock(); + catch (Exception ex) { + log.warn(ex, "Error, when parse DynamicAllocationTasksNotice"); } } } /** - * This method determines whether and how to do scale actions based on collected lag points. - * Current algorithm of scale is simple: - * First of all, compute the proportion of lag points higher/lower than scaleOutThreshold/scaleInThreshold, getting scaleOutThreshold/scaleInThreshold. - * Secondly, compare scaleOutThreshold/scaleInThreshold with triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency. P.S. Scale out action has higher priority than scale in action. - * Finaly, if scaleOutThreshold/scaleInThreshold is higher than triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency, scale out/in action would be triggered. + * This method determines how to do scale actions based on collected lag points. * If scale action is triggered : * First of all, call gracefulShutdownInternal() which will change the state of current datasource ingest tasks from reading to publishing. * Secondly, clear all the stateful data structures: activelyReadingTaskGroups, partitionGroups, partitionOffsets, pendingCompletionTaskGroups, partitionIds. These structures will be rebuiled next 'RunNotice'. * Finally, change taskCount in SeekableStreamSupervisorIOConfig and sync it to MetaStorage. * After changed taskCount in SeekableStreamSupervisorIOConfig, next RunNotice will ceate scaled number of ingest tasks without resubmitting supervisors. - * @param lags the lag metrics of Stream(Kafka/Kinesis) + * @param desireActiveTaskCount desire taskCount compute from autoscaler * @return Boolean flag, do scale action successfully or not. If true , it will take at least 'minTriggerDynamicFrequency' before next 'dynamicAllocatie'. * If false, it will do 'dynamicAllocate' again after 'dynamicCheckPeriod'. * @throws InterruptedException * @throws ExecutionException * @throws TimeoutException */ - private boolean dynamicAllocate(List lags) throws InterruptedException, ExecutionException, TimeoutException + private boolean dynamicAllocate(Integer desireActiveTaskCount) throws InterruptedException, ExecutionException, TimeoutException { - // if supervisor is not suspended, ensure required tasks are running - // if suspended, ensure tasks have been requested to gracefully stop - log.info("[%s] supervisor is running, start to check dynamic allocate task logic. Current collected lags : [%s]", dataSource, lags); - int beyond = 0; - int within = 0; - int metricsCount = lags.size(); - for (Long lag : lags) { - if (lag >= scaleOutThreshold) { - beyond++; - } - if (lag <= scaleInThreshold) { - within++; - } - } - double beyondProportion = beyond * 1.0 / metricsCount; - double withinProportion = within * 1.0 / metricsCount; - log.debug("triggerScaleOutThresholdFrequency is [%s] and triggerScaleInThresholdFrequency is [%s] for dataSource [%s].", triggerScaleOutThresholdFrequency, triggerScaleInThresholdFrequency, dataSource); - log.info("beyondProportion is [%s] and withinProportion is [%s] for dataSource [%s].", beyondProportion, withinProportion, dataSource); - int currentActiveTaskCount; - int desireActiveTaskCount; Collection activeTaskGroups = activelyReadingTaskGroups.values(); currentActiveTaskCount = activeTaskGroups.size(); - if (beyondProportion >= triggerScaleOutThresholdFrequency) { - // Do Scale out - int taskCount = currentActiveTaskCount + scaleOutStep; - if (currentActiveTaskCount == taskCountMax) { - log.info("CurrentActiveTaskCount reach task count Max limit, skip to scale out tasks for dataSource [%s].", dataSource); - return false; - } else { - desireActiveTaskCount = Math.min(taskCount, taskCountMax); - } - log.debug("Start to scale out tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desireActiveTaskCount, dataSource); - gracefulShutdownInternal(); - // clear everything - clearAllocationInfos(); - log.info("Change taskCount to [%s] for dataSource [%s].", desireActiveTaskCount, dataSource); - changeTaskCountInIOConfig(desireActiveTaskCount); - return true; - } - - if (withinProportion >= triggerScaleInThresholdFrequency) { - // Do Scale in - int taskCount = currentActiveTaskCount - scaleInStep; - if (currentActiveTaskCount == taskCountMin) { - log.info("CurrentActiveTaskCount reach task count Min limit, skip to scale in tasks for dataSource [%s].", dataSource); - return false; - } else { - desireActiveTaskCount = Math.max(taskCount, taskCountMin); - } - log.debug("Start to scale in tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desireActiveTaskCount, dataSource); + if (desireActiveTaskCount == -1) { + return false; + } else { + log.debug("Start to scale action tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desireActiveTaskCount, dataSource); gracefulShutdownInternal(); // clear everything clearAllocationInfos(); @@ -450,7 +401,6 @@ private boolean dynamicAllocate(List lags) throws InterruptedException, Ex changeTaskCountInIOConfig(desireActiveTaskCount); return true; } - return false; } private void changeTaskCountInIOConfig(int desireActiveTaskCount) @@ -480,11 +430,6 @@ private void clearAllocationInfos() partitionIds.clear(); } - private List collectTotalLags() - { - return new ArrayList<>(lagMetricsQueue); - } - private class GracefulShutdownNotice extends ShutdownNotice { @Override @@ -647,8 +592,6 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private final ExecutorService exec; private final ScheduledExecutorService scheduledExec; private final ScheduledExecutorService reportingExec; - private final ScheduledExecutorService allocationExec; - private final ScheduledExecutorService lagComputationExec; private final ListeningExecutorService workerExec; private final BlockingQueue notices = new LinkedBlockingDeque<>(); private final Object stopLock = new Object(); @@ -668,22 +611,9 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private volatile boolean lifecycleStarted = false; private final ServiceEmitter emitter; private final boolean enableDynamicAllocationTasks; - private long metricsCollectionIntervalMillis; - private long metricsCollectionRangeMillis; - private long scaleOutThreshold; - private double triggerScaleOutThresholdFrequency; - private long scaleInThreshold; - private double triggerScaleInThresholdFrequency; - private long dynamicCheckStartDelayMillis; - private long dynamicCheckPeriod; private int taskCountMax; - private int taskCountMin; - private int scaleInStep; - private int scaleOutStep; private long minTriggerDynamicFrequency; - private volatile CircularFifoQueue lagMetricsQueue; - public SeekableStreamSupervisor( final String supervisorId, final TaskStorage taskStorage, @@ -710,30 +640,13 @@ public SeekableStreamSupervisor( log.debug("Get dynamicAllocationTasksProperties from IOConfig : [%s] in [%s]", dynamicAllocationTasksProperties, dataSource); if (dynamicAllocationTasksProperties != null && !dynamicAllocationTasksProperties.isEmpty() && Boolean.parseBoolean(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("enableDynamicAllocationTasks", false)))) { log.info("EnableDynamicAllocationTasks for datasource [%s]", dataSource); + this.taskCountMax = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 4))); + this.minTriggerDynamicFrequency = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("minTriggerDynamicFrequencyMillis", 600000))); this.enableDynamicAllocationTasks = true; } else { log.info("Disable dynamic allocate tasks for [%s]", dataSource); this.enableDynamicAllocationTasks = false; } - if (enableDynamicAllocationTasks) { - this.metricsCollectionIntervalMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionIntervalMillis", 30000))); - this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionRangeMillis", 600000))); - int slots = (int) (metricsCollectionRangeMillis / metricsCollectionIntervalMillis) + 1; - log.debug(" The interval of metrics collection is [%s], [%s] timeRange will collect [%s] data points for dataSource [%s].", metricsCollectionIntervalMillis, metricsCollectionRangeMillis, slots, dataSource); - this.lagMetricsQueue = new CircularFifoQueue<>(slots); - this.dynamicCheckStartDelayMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckStartDelayMillis", 300000))); - this.dynamicCheckPeriod = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckPeriod", 60000))); - this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionRangeMillis", 600000))); - this.scaleOutThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutThreshold", 6000000))); - this.scaleInThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInThreshold", 1000000))); - this.triggerScaleOutThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerScaleOutThresholdFrequency", 0.3))); - this.triggerScaleInThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerScaleInThresholdFrequency", 0.9))); - this.taskCountMax = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 4))); - this.taskCountMin = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMin", 1))); - this.scaleInStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInStep", 1))); - this.scaleOutStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutStep", 2))); - this.minTriggerDynamicFrequency = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("minTriggerDynamicFrequencyMillis", 600000))); - } this.tuningConfig = spec.getTuningConfig(); this.taskTuningConfig = this.tuningConfig.convertToTaskTuningConfig(); @@ -741,8 +654,6 @@ public SeekableStreamSupervisor( this.exec = Execs.singleThreaded(StringUtils.encodeForFormat(supervisorId)); this.scheduledExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Scheduler-%d"); this.reportingExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Reporting-%d"); - this.allocationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Allocation-%d"); - this.lagComputationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Computation-%d"); this.stateManager = new SeekableStreamSupervisorStateManager( spec.getSupervisorStateManagerConfig(), @@ -830,6 +741,13 @@ public Optional getTaskStatus(String id) ); } + + @Override + public Map getSupervisorTaskInfos() + { + return activelyReadingTaskGroups; + } + @Override public void start() { @@ -891,8 +809,6 @@ public void stop(boolean stopGracefully) try { scheduledExec.shutdownNow(); // stop recurring executions reportingExec.shutdownNow(); - allocationExec.shutdownNow(); - lagComputationExec.shutdownNow(); if (started) { @@ -1010,22 +926,6 @@ public void tryInit() ); scheduleReporting(reportingExec); - if (enableDynamicAllocationTasks) { - log.debug("Collect and compute lags at fixed rate of [%s] for dataSource[%s].", metricsCollectionIntervalMillis, dataSource); - lagComputationExec.scheduleAtFixedRate( - collectAndComputeLags(), - dynamicCheckStartDelayMillis, // wait for tasks to start up - metricsCollectionIntervalMillis, - TimeUnit.MILLISECONDS - ); - log.debug("allocate task at fixed rate of [%s], dataSource [%s].", dynamicCheckPeriod, dataSource); - allocationExec.scheduleAtFixedRate( - buildDynamicAllocationTask(), - dynamicCheckStartDelayMillis + metricsCollectionRangeMillis, - dynamicCheckPeriod, - TimeUnit.MILLISECONDS - ); - } started = true; log.info( "Started SeekableStreamSupervisor[%s], first run in [%s], with spec: [%s]", @@ -1048,40 +948,10 @@ public void tryInit() } } - /** - * This method compute current consume lags. Get the total lags of all partition and fill in lagMetricsQueue - * @return a Runnbale object to do collect and compute action. - */ - private Runnable collectAndComputeLags() - { - return new Runnable() { - @Override - public void run() - { - lock.lock(); - try { - if (!spec.isSuspended()) { - ArrayList metricsInfo = new ArrayList<>(3); - collectLag(metricsInfo); - long totalLags = metricsInfo.size() < 3 ? 0 : metricsInfo.get(1); - lagMetricsQueue.offer(totalLags > 0 ? totalLags : 0); - log.debug("Current lag metric points [%s] for dataSource [%s].", new ArrayList<>(lagMetricsQueue), dataSource); - } else { - log.debug("[%s] supervisor is suspended, skip to collect kafka lags", dataSource); - } - } - catch (Exception e) { - log.warn(e, "Error, When collect kafka lags"); - } - finally { - lock.unlock(); - } - } - }; - } - private Runnable buildDynamicAllocationTask() + @Override + public Runnable buildDynamicAllocationTask(Callable scaleAction) { - return () -> notices.add(new DynamicAllocationTasksNotice()); + return () -> notices.add(new DynamicAllocationTasksNotice(scaleAction)); } private Runnable buildRunTask() @@ -1431,20 +1301,6 @@ public void gracefulShutdownInternal() throws ExecutionException, InterruptedExc @VisibleForTesting public void resetInternal(DataSourceMetadata dataSourceMetadata) { - // clear queue for kafka lags - if (enableDynamicAllocationTasks && lagMetricsQueue != null) { - try { - lock.lock(); - lagMetricsQueue.clear(); - } - catch (Exception e) { - log.warn(e, "Error,when clear queue in rest action"); - } - finally { - lock.unlock(); - } - } - if (dataSourceMetadata == null) { // Reset everything boolean result = indexerMetadataStorageCoordinator.deleteDataSourceMetadata(dataSource); @@ -3892,11 +3748,4 @@ protected void computeLags(Map partitionLags, ArrayList lags - * Only support Kafka ingestion so far. - * @param lags , Notice : The order of values is maxLag, totalLag and avgLag. - */ - protected abstract void collectLag(ArrayList lags); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java index 50be4649bccf..142ed39eb542 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java @@ -32,6 +32,9 @@ import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.DefaultAutoScaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.DummyAutoScaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientFactory; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; @@ -151,6 +154,27 @@ public DruidMonitorSchedulerConfig getMonitorSchedulerConfig() @Override public abstract Supervisor createSupervisor(); + /** + * need to notice that autoScaler would be null which means autoscale is dissable. + * @param supervisor + * @return + */ + @Override + public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + { + String dataSource = getId(); + SupervisorTaskAutoscaler autoScaler = new DummyAutoScaler(supervisor, dataSource); + Map dynamicAllocationTasksProperties = ingestionSchema.getIOConfig().getDynamicAllocationTasksProperties(); + if (dynamicAllocationTasksProperties != null && !dynamicAllocationTasksProperties.isEmpty() && Boolean.parseBoolean(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("enableDynamicAllocationTasks", false)))) { + String autoScalerStrategy = String.valueOf(dynamicAllocationTasksProperties.getOrDefault("autoScalerStrategy", "lagBased")); + + switch (autoScalerStrategy) { + default: autoScaler = new DefaultAutoScaler(supervisor, dataSource, dynamicAllocationTasksProperties, this); + } + } + return autoScaler; + } + @Override public List getDataSources() { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java index e603a553e244..4b831a9607e1 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java @@ -32,6 +32,8 @@ import org.apache.druid.indexing.overlord.supervisor.SupervisorManager; import org.apache.druid.indexing.overlord.supervisor.SupervisorResource; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.DummyAutoScaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.indexing.worker.http.WorkerResource; import org.apache.druid.server.http.security.ResourceFilterTestHelper; import org.apache.druid.server.security.AuthorizerMapper; @@ -126,6 +128,12 @@ public Supervisor createSupervisor() return null; } + @Override + public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + { + return new DummyAutoScaler(null, null); + } + @Override public List getDataSources() { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java index 6b93f84d1625..82e98167ffb6 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java @@ -22,6 +22,7 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; import org.apache.druid.indexing.overlord.DataSourceMetadata; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.metadata.MetadataSupervisorManager; import org.easymock.Capture; @@ -404,6 +405,12 @@ public Supervisor createSupervisor() return supervisor; } + @Override + public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + { + return null; + } + @Override public boolean isSuspended() { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorResourceTest.java index 1390e7acc98c..e714798f01b5 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorResourceTest.java @@ -26,6 +26,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.druid.indexing.overlord.DataSourceMetadata; import org.apache.druid.indexing.overlord.TaskMaster; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.segment.TestHelper; import org.apache.druid.server.security.Access; @@ -1155,6 +1156,12 @@ public Supervisor createSupervisor() return supervisor; } + @Override + public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + { + return null; + } + @Override public List getDataSources() { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index 7e6067f03a68..2b1c435ce15f 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -1201,7 +1201,7 @@ protected void scheduleReporting(ScheduledExecutorService reportingExec) } @Override - protected void collectLag(ArrayList lags) + public void collectLag(ArrayList lags) { } } @@ -1247,7 +1247,7 @@ protected void emitLag() } @Override - protected void collectLag(ArrayList lags) + public void collectLag(ArrayList lags) { } diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java index a2aa29f9c25c..e128e3766e8e 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java @@ -23,11 +23,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import org.apache.druid.indexing.overlord.DataSourceMetadata; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.concurrent.Callable; /** * Used as a tombstone marker in the supervisors metadata table to indicate that the supervisor has been removed. @@ -154,9 +157,32 @@ public void checkpoint(int taskGroupId, DataSourceMetadata checkpointMetadata) { } + + @Override + public void collectLag(ArrayList lags) + { + } + + @Override + public Runnable buildDynamicAllocationTask(Callable scaleAction) + { + return null; + } + + @Override + public Map getSupervisorTaskInfos() + { + return null; + } }; } + @Override + public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + { + return null; + } + @Override public SupervisorSpec createRunningSpec() { diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java index 83c661a12ed8..b4df1c2c12f2 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java @@ -23,7 +23,9 @@ import org.apache.druid.indexing.overlord.DataSourceMetadata; import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Map; +import java.util.concurrent.Callable; public interface Supervisor { @@ -64,4 +66,18 @@ default Boolean isHealthy() * @param checkpointMetadata metadata for the sequence to currently checkpoint */ void checkpoint(int taskGroupId, DataSourceMetadata checkpointMetadata); + + /** + * Collect maxLag, totalLag, avgLag into ArrayList lags + * Only support Kafka ingestion so far. + * @param lags , Notice : The order of values is maxLag, totalLag and avgLag. + */ + void collectLag(ArrayList lags); + + /** + * use for autoscaler + */ + Runnable buildDynamicAllocationTask(Callable scaleAction); + + Map getSupervisorTaskInfos(); } diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java index 041156ce4251..adc462f42ce9 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import java.util.List; @@ -40,6 +41,8 @@ public interface SupervisorSpec */ Supervisor createSupervisor(); + SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor); + List getDataSources(); default SupervisorSpec createSuspendedSpec() diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DefaultAutoScaler.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DefaultAutoScaler.java new file mode 100644 index 000000000000..0ebd86c360ed --- /dev/null +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DefaultAutoScaler.java @@ -0,0 +1,254 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.supervisor.autoscaler; + +import org.apache.commons.collections4.queue.CircularFifoQueue; +import org.apache.druid.indexing.overlord.supervisor.Supervisor; +import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.emitter.EmittingLogger; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +public class DefaultAutoScaler implements SupervisorTaskAutoscaler +{ + private static final EmittingLogger log = new EmittingLogger(DefaultAutoScaler.class); + private final String dataSource; + private final long metricsCollectionIntervalMillis; + private final long metricsCollectionRangeMillis; + private final Map supervisorTaskInfos; + private final CircularFifoQueue lagMetricsQueue; + private final long dynamicCheckStartDelayMillis; + private final long dynamicCheckPeriod; + private final long scaleOutThreshold; + private final long scaleInThreshold; + private final double triggerScaleOutThresholdFrequency; + private final double triggerScaleInThresholdFrequency; + private final int taskCountMax; + private final int taskCountMin; + private final int scaleInStep; + private final int scaleOutStep; + private final ScheduledExecutorService lagComputationExec; + private final ScheduledExecutorService allocationExec; + private final SupervisorSpec spec; + private final Supervisor supervisor; + + private static ReentrantLock lock = new ReentrantLock(true); + + + public DefaultAutoScaler(Supervisor supervisor, String dataSource, Map dynamicAllocationTasksProperties, SupervisorSpec spec) + { + String supervisorId = StringUtils.format("KafkaSupervisor-%s", dataSource); + this.dataSource = dataSource; + this.metricsCollectionIntervalMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionIntervalMillis", 30000))); + this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionRangeMillis", 600000))); + int slots = (int) (metricsCollectionRangeMillis / metricsCollectionIntervalMillis) + 1; + log.debug(" The interval of metrics collection is [%s], [%s] timeRange will collect [%s] data points for dataSource [%s].", metricsCollectionIntervalMillis, metricsCollectionRangeMillis, slots, dataSource); + this.lagMetricsQueue = new CircularFifoQueue<>(slots); + this.dynamicCheckStartDelayMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckStartDelayMillis", 300000))); + this.dynamicCheckPeriod = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckPeriod", 60000))); + this.scaleOutThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutThreshold", 6000000))); + this.scaleInThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInThreshold", 1000000))); + this.triggerScaleOutThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerScaleOutThresholdFrequency", 0.3))); + this.triggerScaleInThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerScaleInThresholdFrequency", 0.9))); + this.taskCountMax = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 4))); + this.taskCountMin = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMin", 1))); + this.scaleInStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInStep", 1))); + this.scaleOutStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutStep", 2))); + this.allocationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Allocation-%d"); + this.lagComputationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Computation-%d"); + this.spec = spec; + this.supervisor = supervisor; + this.supervisorTaskInfos = supervisor.getSupervisorTaskInfos(); + } + + @Override + public void start() + { + Callable scaleAction = new Callable() { + @Override + public Integer call() + { + lock.lock(); + int desireTaskCount = -1; + try { + desireTaskCount = computeDesireTaskCount(new ArrayList<>(lagMetricsQueue)); + + if (desireTaskCount != -1) { + lagMetricsQueue.clear(); + } + } + catch (Exception ex) { + log.warn(ex, "Exception when computeDesireTaskCount [%s]", dataSource); + } + finally { + lock.unlock(); + } + return desireTaskCount; + } + }; + + log.info("EnableDynamicAllocationTasks for datasource [%s]", dataSource); + log.debug("Collect and compute lags at fixed rate of [%s] for dataSource[%s].", metricsCollectionIntervalMillis, dataSource); + lagComputationExec.scheduleAtFixedRate( + collectAndComputeLags(), + dynamicCheckStartDelayMillis, // wait for tasks to start up + metricsCollectionIntervalMillis, + TimeUnit.MILLISECONDS + ); + log.debug("allocate task at fixed rate of [%s], dataSource [%s].", dynamicCheckPeriod, dataSource); + allocationExec.scheduleAtFixedRate( + supervisor.buildDynamicAllocationTask(scaleAction), + dynamicCheckStartDelayMillis + metricsCollectionRangeMillis, + dynamicCheckPeriod, + TimeUnit.MILLISECONDS + ); + } + + @Override + public void stop() + { + allocationExec.shutdownNow(); + lagComputationExec.shutdownNow(); + } + + @Override + public void reset() + { + // clear queue for kafka lags + if (lagMetricsQueue != null) { + try { + lock.lock(); + lagMetricsQueue.clear(); + } + catch (Exception e) { + log.warn(e, "Error,when clear queue in rest action"); + } + finally { + lock.unlock(); + } + } + } + + /** + * This method compute current consume lags. Get the total lags of all partition and fill in lagMetricsQueue + * @return a Runnbale object to do collect and compute action. + */ + private Runnable collectAndComputeLags() + { + return new Runnable() { + @Override + public void run() + { + lock.lock(); + try { + if (!spec.isSuspended()) { + ArrayList metricsInfo = new ArrayList<>(3); + supervisor.collectLag(metricsInfo); + long totalLags = metricsInfo.size() < 3 ? 0 : metricsInfo.get(1); + lagMetricsQueue.offer(totalLags > 0 ? totalLags : 0); + log.debug("Current lag metric points [%s] for dataSource [%s].", new ArrayList<>(lagMetricsQueue), dataSource); + } else { + log.debug("[%s] supervisor is suspended, skip to collect kafka lags", dataSource); + } + } + catch (Exception e) { + log.warn(e, "Error, When collect kafka lags"); + } + finally { + lock.unlock(); + } + } + }; + } + + /** + * This method determines whether to do scale actions based on collected lag points. + * Current algorithm of scale is simple: + * First of all, compute the proportion of lag points higher/lower than scaleOutThreshold/scaleInThreshold, getting scaleOutThreshold/scaleInThreshold. + * Secondly, compare scaleOutThreshold/scaleInThreshold with triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency. P.S. Scale out action has higher priority than scale in action. + * Finaly, if scaleOutThreshold/scaleInThreshold is higher than triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency, scale out/in action would be triggered. + * @param lags the lag metrics of Stream(Kafka/Kinesis) + * @return Boolean flag, do scale action successfully or not. If true , it will take at least 'minTriggerDynamicFrequency' before next 'dynamicAllocatie'. + * If false, it will do 'dynamicAllocate' again after 'dynamicCheckPeriod'. + * + * @return Integer. target number of tasksCount, -1 means skip scale action. + */ + private Integer computeDesireTaskCount(List lags) + { + // if supervisor is not suspended, ensure required tasks are running + // if suspended, ensure tasks have been requested to gracefully stop + log.info("[%s] supervisor is running, start to check dynamic allocate task logic. Current collected lags : [%s]", dataSource, lags); + int beyond = 0; + int within = 0; + int metricsCount = lags.size(); + for (Long lag : lags) { + if (lag >= scaleOutThreshold) { + beyond++; + } + if (lag <= scaleInThreshold) { + within++; + } + } + double beyondProportion = beyond * 1.0 / metricsCount; + double withinProportion = within * 1.0 / metricsCount; + log.debug("triggerScaleOutThresholdFrequency is [%s] and triggerScaleInThresholdFrequency is [%s] for dataSource [%s].", triggerScaleOutThresholdFrequency, triggerScaleInThresholdFrequency, dataSource); + log.info("beyondProportion is [%s] and withinProportion is [%s] for dataSource [%s].", beyondProportion, withinProportion, dataSource); + + int currentActiveTaskCount; + int desireActiveTaskCount; + currentActiveTaskCount = supervisorTaskInfos.values().size(); + + if (beyondProportion >= triggerScaleOutThresholdFrequency) { + // Do Scale out + int taskCount = currentActiveTaskCount + scaleOutStep; + if (currentActiveTaskCount == taskCountMax) { + log.info("CurrentActiveTaskCount reach task count Max limit, skip to scale out tasks for dataSource [%s].", dataSource); + return -1; + } else { + desireActiveTaskCount = Math.min(taskCount, taskCountMax); + } + + return desireActiveTaskCount; + } + + if (withinProportion >= triggerScaleInThresholdFrequency) { + // Do Scale in + int taskCount = currentActiveTaskCount - scaleInStep; + if (currentActiveTaskCount == taskCountMin) { + log.info("CurrentActiveTaskCount reach task count Min limit, skip to scale in tasks for dataSource [%s].", dataSource); + return -1; + } else { + desireActiveTaskCount = Math.max(taskCount, taskCountMin); + } + log.debug("Start to scale in tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desireActiveTaskCount, dataSource); + return desireActiveTaskCount; + } + + return -1; + } +} diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DummyAutoScaler.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DummyAutoScaler.java new file mode 100644 index 000000000000..8f13e66a93b6 --- /dev/null +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DummyAutoScaler.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.supervisor.autoscaler; + +import org.apache.druid.indexing.overlord.supervisor.Supervisor; + +public class DummyAutoScaler implements SupervisorTaskAutoscaler +{ + public DummyAutoScaler(Supervisor supervisor, String dataSource) + { + } + + @Override + public void start() + { + //Do nothing + } + + @Override + public void stop() + { + //Do nothing + } + + @Override + public void reset() + { + //Do nothing + } +} diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/SupervisorTaskAutoscaler.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/SupervisorTaskAutoscaler.java new file mode 100644 index 000000000000..ecc1d2810316 --- /dev/null +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/SupervisorTaskAutoscaler.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.supervisor.autoscaler; + +public interface SupervisorTaskAutoscaler +{ + void start(); + void stop(); + void reset(); +} diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java index 93322209c8a2..624157a52552 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java @@ -25,6 +25,8 @@ import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; import org.apache.druid.indexing.overlord.supervisor.VersionedSupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.DummyAutoScaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.StringUtils; import org.junit.After; @@ -185,6 +187,12 @@ public Supervisor createSupervisor() throw new UnsupportedOperationException(); } + @Override + public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + { + return new DummyAutoScaler(supervisor, null); + } + @Override public List getDataSources() { diff --git a/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java b/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java index e935a413f045..407520a0968d 100644 --- a/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java +++ b/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java @@ -23,6 +23,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.DummyAutoScaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import java.util.List; import java.util.Objects; @@ -52,6 +54,12 @@ public Supervisor createSupervisor() return null; } + @Override + public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + { + return new DummyAutoScaler(supervisor, null); + } + @Override public List getDataSources() { From 76db5ba009fbd54d5b45ffcc3193d3c05373fbe0 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Sat, 16 Jan 2021 15:52:06 +0800 Subject: [PATCH 17/47] fix ci failed --- .idea/misc.xml | 4 ++-- indexing-service/pom.xml | 5 ----- .../overlord/supervisor/SupervisorManager.java | 2 +- .../supervisor/SeekableStreamSupervisorSpec.java | 10 +++++++--- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 1fbd11f67270..33f4df5d043e 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -67,11 +67,6 @@ io.dropwizard.metrics metrics-core - - org.apache.commons - commons-collections4 - 4.2 - com.google.code.findbugs jsr305 diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java index bf678193d218..bf502473e3ba 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java @@ -314,6 +314,7 @@ private boolean createAndStartSupervisorInternal(SupervisorSpec spec, boolean pe supervisor.start(); if (autoscaler != null) { autoscaler.start(); + autoscalers.put(id, autoscaler); } } catch (Exception e) { @@ -325,7 +326,6 @@ private boolean createAndStartSupervisorInternal(SupervisorSpec spec, boolean pe } supervisors.put(id, Pair.of(supervisor, spec)); - autoscalers.put(id, autoscaler); return true; } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java index 142ed39eb542..6e08aa0e0298 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java @@ -25,6 +25,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import org.apache.druid.annotations.SuppressFBWarnings; import org.apache.druid.guice.annotations.Json; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.TaskMaster; @@ -36,6 +37,7 @@ import org.apache.druid.indexing.overlord.supervisor.autoscaler.DummyAutoScaler; import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientFactory; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; import org.apache.druid.segment.indexing.DataSchema; @@ -157,18 +159,20 @@ public DruidMonitorSchedulerConfig getMonitorSchedulerConfig() /** * need to notice that autoScaler would be null which means autoscale is dissable. * @param supervisor - * @return + * @return autoScaler, disable autoscale will return dummyAutoScaler and enable autoscale wiil return defaultAutoScaler by default. */ @Override + @SuppressFBWarnings(value = "RV_RETURN_VALUE_IGNORED", justification = "using siwtch(String)") public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) { String dataSource = getId(); SupervisorTaskAutoscaler autoScaler = new DummyAutoScaler(supervisor, dataSource); Map dynamicAllocationTasksProperties = ingestionSchema.getIOConfig().getDynamicAllocationTasksProperties(); if (dynamicAllocationTasksProperties != null && !dynamicAllocationTasksProperties.isEmpty() && Boolean.parseBoolean(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("enableDynamicAllocationTasks", false)))) { - String autoScalerStrategy = String.valueOf(dynamicAllocationTasksProperties.getOrDefault("autoScalerStrategy", "lagBased")); + String autoScalerStrategy = String.valueOf(dynamicAllocationTasksProperties.getOrDefault("autoScalerStrategy", "default")); - switch (autoScalerStrategy) { + // will thorw 'Return value of String.hashCode() ignored : RV_RETURN_VALUE_IGNORED' just Suppress it. + switch (StringUtils.toLowerCase(autoScalerStrategy)) { default: autoScaler = new DefaultAutoScaler(supervisor, dataSource, dynamicAllocationTasksProperties, this); } } From 751175fc7cf449fe4adf533c3245878e483a70e6 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Sat, 16 Jan 2021 15:53:16 +0800 Subject: [PATCH 18/47] revert msic.xml --- .idea/misc.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + From f8a67072ad8deeab6e1035589e568552cd5eec0c Mon Sep 17 00:00:00 2001 From: yuezhang Date: Sun, 17 Jan 2021 06:33:41 +0800 Subject: [PATCH 19/47] add uts to test autoscaler create && scale out/in and kafka ingest with scale enable --- .../kafka/supervisor/KafkaSupervisorTest.java | 194 +++++ .../SeekableStreamSupervisorSpecTest.java | 786 ++++++++++++++++++ 2 files changed, 980 insertions(+) create mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 9c182d6dfb20..4ee25c281831 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -61,11 +61,13 @@ import org.apache.druid.indexing.overlord.supervisor.SupervisorReport; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.indexing.seekablestream.SeekableStreamEndSequenceNumbers; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.Status; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskTuningConfig; import org.apache.druid.indexing.seekablestream.SeekableStreamStartSequenceNumbers; import org.apache.druid.indexing.seekablestream.common.RecordSupplier; +import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorSpec; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager; import org.apache.druid.indexing.seekablestream.supervisor.TaskReportData; import org.apache.druid.java.util.common.DateTimes; @@ -158,6 +160,7 @@ public class KafkaSupervisorTest extends EasyMockSupport private RowIngestionMetersFactory rowIngestionMetersFactory; private ExceptionCapturingServiceEmitter serviceEmitter; private SupervisorStateManagerConfig supervisorConfig; + private KafkaSupervisorIngestionSpec ingestionSchema; private static String getTopic() { @@ -214,6 +217,7 @@ public void setupTest() serviceEmitter = new ExceptionCapturingServiceEmitter(); EmittingLogger.registerEmitter(serviceEmitter); supervisorConfig = new SupervisorStateManagerConfig(); + ingestionSchema = EasyMock.createMock(KafkaSupervisorIngestionSpec.class); } @After @@ -232,6 +236,196 @@ public static void tearDownClass() throws IOException zkServer = null; } + @Test + public void testNoInitialStateWithAutoscaler() throws Exception + { + KafkaIndexTaskClientFactory taskClientFactory = new KafkaIndexTaskClientFactory( + null, + null + ) + { + @Override + public KafkaIndexTaskClient build( + TaskInfoProvider taskInfoProvider, + String dataSource, + int numThreads, + Duration httpTimeout, + long numRetries + ) + { + Assert.assertEquals(TEST_CHAT_THREADS, numThreads); + Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), httpTimeout); + Assert.assertEquals(TEST_CHAT_RETRIES, numRetries); + return taskClient; + } + }; + + HashMap dynamicAllocationTasksProperties = new HashMap<>(); + dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", true); + dynamicAllocationTasksProperties.put("metricsCollectionIntervalMillis", 500); + dynamicAllocationTasksProperties.put("metricsCollectionRangeMillis", 500); + dynamicAllocationTasksProperties.put("scaleOutThreshold", 0); + dynamicAllocationTasksProperties.put("triggerScaleOutThresholdFrequency", 0.0); + dynamicAllocationTasksProperties.put("scaleInThreshold", 1000000); + dynamicAllocationTasksProperties.put("triggerScaleInThresholdFrequency", 0.8); + dynamicAllocationTasksProperties.put("dynamicCheckStartDelayMillis", 0); + dynamicAllocationTasksProperties.put("dynamicCheckPeriod", 100); + dynamicAllocationTasksProperties.put("taskCountMax", 2); + dynamicAllocationTasksProperties.put("taskCountMin", 1); + dynamicAllocationTasksProperties.put("scaleInStep", 1); + dynamicAllocationTasksProperties.put("scaleOutStep", 2); + dynamicAllocationTasksProperties.put("minTriggerDynamicFrequencyMillis", 1200000); + + final Map consumerProperties = KafkaConsumerConfigs.getConsumerProperties(); + consumerProperties.put("myCustomKey", "myCustomValue"); + consumerProperties.put("bootstrap.servers", kafkaHost); + + KafkaSupervisorIOConfig kafkaSupervisorIOConfig = new KafkaSupervisorIOConfig( + topic, + INPUT_FORMAT, + 1, + 1, + new Period("PT1H"), + consumerProperties, + dynamicAllocationTasksProperties, + KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, + new Period("P1D"), + new Period("PT30S"), + true, + new Period("PT30M"), + null, + null, + null + ); + + final KafkaSupervisorTuningConfig tuningConfigOri = new KafkaSupervisorTuningConfig( + null, + 1000, + null, + 50000, + null, + new Period("P1Y"), + new File("/test"), + null, + null, + null, + true, + false, + null, + false, + null, + numThreads, + TEST_CHAT_THREADS, + TEST_CHAT_RETRIES, + TEST_HTTP_TIMEOUT, + TEST_SHUTDOWN_TIMEOUT, + null, + null, + null, + null, + null + ); + + EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(kafkaSupervisorIOConfig).anyTimes(); + EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes(); + EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(tuningConfigOri).anyTimes(); + EasyMock.replay(ingestionSchema); + + SeekableStreamSupervisorSpec testableSupervisorSpec = new KafkaSupervisorSpec( + ingestionSchema, + dataSchema, + tuningConfigOri, + kafkaSupervisorIOConfig, + null, + false, + taskStorage, + taskMaster, + indexerMetadataStorageCoordinator, + taskClientFactory, + OBJECT_MAPPER, + new NoopServiceEmitter(), + new DruidMonitorSchedulerConfig(), + rowIngestionMetersFactory, + new SupervisorStateManagerConfig() + ); + + supervisor = new TestableKafkaSupervisor( + taskStorage, + taskMaster, + indexerMetadataStorageCoordinator, + taskClientFactory, + OBJECT_MAPPER, + (KafkaSupervisorSpec) testableSupervisorSpec, + rowIngestionMetersFactory + ); + + SupervisorTaskAutoscaler autoscaler = testableSupervisorSpec.createAutoscaler(supervisor); + + + final KafkaSupervisorTuningConfig tuningConfig = supervisor.getTuningConfig(); + addSomeEvents(1); + + Capture captured = Capture.newInstance(); + EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes(); + EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes(); + EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes(); + EasyMock.expect(taskStorage.getActiveTasksByDatasource(DATASOURCE)).andReturn(ImmutableList.of()).anyTimes(); + EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn( + new KafkaDataSourceMetadata( + null + ) + ).anyTimes(); + EasyMock.expect(taskQueue.add(EasyMock.capture(captured))).andReturn(true); + taskRunner.registerListener(EasyMock.anyObject(TaskRunnerListener.class), EasyMock.anyObject(Executor.class)); + replayAll(); + + supervisor.start(); + int taskCountBeforeScale = supervisor.getIoConfig().getTaskCount(); + Assert.assertEquals(1, taskCountBeforeScale); + autoscaler.start(); + supervisor.runInternal(); + Thread.sleep(1 * 1000); + verifyAll(); + + int taskCountAfterScale = supervisor.getIoConfig().getTaskCount(); + Assert.assertEquals(2, taskCountAfterScale); + + + KafkaIndexTask task = captured.getValue(); + Assert.assertEquals(KafkaSupervisorTest.dataSchema, task.getDataSchema()); + Assert.assertEquals(tuningConfig.convertToTaskTuningConfig(), task.getTuningConfig()); + + KafkaIndexTaskIOConfig taskConfig = task.getIOConfig(); + Assert.assertEquals(kafkaHost, taskConfig.getConsumerProperties().get("bootstrap.servers")); + Assert.assertEquals("myCustomValue", taskConfig.getConsumerProperties().get("myCustomKey")); + Assert.assertEquals("sequenceName-0", taskConfig.getBaseSequenceName()); + Assert.assertTrue("isUseTransaction", taskConfig.isUseTransaction()); + Assert.assertFalse("minimumMessageTime", taskConfig.getMinimumMessageTime().isPresent()); + Assert.assertFalse("maximumMessageTime", taskConfig.getMaximumMessageTime().isPresent()); + + Assert.assertEquals(topic, taskConfig.getStartSequenceNumbers().getStream()); + Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(0)); + Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(1)); + Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(2)); + + Assert.assertEquals(topic, taskConfig.getEndSequenceNumbers().getStream()); + Assert.assertEquals( + Long.MAX_VALUE, + (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(0) + ); + Assert.assertEquals( + Long.MAX_VALUE, + (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(1) + ); + Assert.assertEquals( + Long.MAX_VALUE, + (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(2) + ); + + autoscaler.reset(); + autoscaler.stop(); + } + @Test public void testCreateBaseTaskContexts() throws JsonProcessingException { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java new file mode 100644 index 000000000000..37611a13c1b8 --- /dev/null +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -0,0 +1,786 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.seekablestream; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.impl.DimensionSchema; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.JsonInputFormat; +import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.indexing.common.task.Task; +import org.apache.druid.indexing.overlord.DataSourceMetadata; +import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; +import org.apache.druid.indexing.overlord.TaskMaster; +import org.apache.druid.indexing.overlord.TaskStorage; +import org.apache.druid.indexing.overlord.supervisor.Supervisor; +import org.apache.druid.indexing.overlord.supervisor.SupervisorManager; +import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.DefaultAutoScaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.DummyAutoScaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.seekablestream.common.OrderedSequenceNumber; +import org.apache.druid.indexing.seekablestream.common.RecordSupplier; +import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisor; +import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorIOConfig; +import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorIngestionSpec; +import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorReportPayload; +import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorSpec; +import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorTuningConfig; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.parsers.JSONPathSpec; +import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.query.aggregation.AggregatorFactory; +import org.apache.druid.query.aggregation.CountAggregatorFactory; +import org.apache.druid.segment.TestHelper; +import org.apache.druid.segment.incremental.RowIngestionMetersFactory; +import org.apache.druid.segment.indexing.DataSchema; +import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec; +import org.apache.druid.server.metrics.DruidMonitorSchedulerConfig; +import org.easymock.EasyMock; +import org.easymock.EasyMockSupport; +import org.easymock.Mock; +import org.joda.time.DateTime; +import org.joda.time.Duration; +import org.joda.time.Period; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.io.File; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.ScheduledExecutorService; + +public class SeekableStreamSupervisorSpecTest extends EasyMockSupport +{ + private SeekableStreamSupervisorIngestionSpec ingestionSchema; + private TaskStorage taskStorage; + private TaskMaster taskMaster; + private IndexerMetadataStorageCoordinator indexerMetadataStorageCoordinator; + private ServiceEmitter emitter; + private RowIngestionMetersFactory rowIngestionMetersFactory; + private DataSchema dataSchema; + private SeekableStreamSupervisorTuningConfig seekableStreamSupervisorTuningConfig; + private SeekableStreamSupervisorIOConfig seekableStreamSupervisorIOConfig; + private static final ObjectMapper OBJECT_MAPPER = TestHelper.makeJsonMapper(); + private SeekableStreamIndexTaskClientFactory taskClientFactory; + private static final String STREAM = "stream"; + private static final String DATASOURCE = "testDS"; + private SeekableStreamSupervisorSpec spec; + private SupervisorStateManagerConfig supervisorConfig; + + private SeekableStreamSupervisor supervisor4; + + private SeekableStreamIndexTaskClientFactory indexTaskClientFactory; + private ObjectMapper mapper; + private DruidMonitorSchedulerConfig monitorSchedulerConfig; + private SupervisorStateManagerConfig supervisorStateManagerConfig; + + @Before + public void setUp() + { + ingestionSchema = EasyMock.mock(SeekableStreamSupervisorIngestionSpec.class); + taskStorage = EasyMock.mock(TaskStorage.class); + taskMaster = EasyMock.mock(TaskMaster.class); + indexerMetadataStorageCoordinator = EasyMock.mock(IndexerMetadataStorageCoordinator.class); + emitter = EasyMock.mock(ServiceEmitter.class); + rowIngestionMetersFactory = EasyMock.mock(RowIngestionMetersFactory.class); + dataSchema = EasyMock.mock(DataSchema.class); + seekableStreamSupervisorTuningConfig = EasyMock.mock(SeekableStreamSupervisorTuningConfig.class); + seekableStreamSupervisorIOConfig = EasyMock.mock(SeekableStreamSupervisorIOConfig.class); + taskClientFactory = EasyMock.mock(SeekableStreamIndexTaskClientFactory.class); + spec = EasyMock.mock(SeekableStreamSupervisorSpec.class); + supervisorConfig = new SupervisorStateManagerConfig(); + indexTaskClientFactory = EasyMock.mock(SeekableStreamIndexTaskClientFactory.class); + mapper = EasyMock.mock(ObjectMapper.class); + monitorSchedulerConfig = EasyMock.mock(DruidMonitorSchedulerConfig.class); + supervisorStateManagerConfig = EasyMock.mock(SupervisorStateManagerConfig.class); + supervisor4 = EasyMock.mock(SeekableStreamSupervisor.class); + } + + private abstract class BaseTestSeekableStreamSupervisor extends SeekableStreamSupervisor + { + private BaseTestSeekableStreamSupervisor() + { + super( + "testSupervisorId", + taskStorage, + taskMaster, + indexerMetadataStorageCoordinator, + taskClientFactory, + OBJECT_MAPPER, + spec, + rowIngestionMetersFactory, + false + ); + } + + @Override + protected String baseTaskName() + { + return "test"; + } + + @Override + protected void updatePartitionLagFromStream() + { + // do nothing + } + + @Nullable + @Override + protected Map getPartitionRecordLag() + { + return null; + } + + @Nullable + @Override + protected Map getPartitionTimeLag() + { + return null; + } + + @Override + protected SeekableStreamIndexTaskIOConfig createTaskIoConfig( + int groupId, + Map startPartitions, + Map endPartitions, + String baseSequenceName, + DateTime minimumMessageTime, + DateTime maximumMessageTime, + Set exclusiveStartSequenceNumberPartitions, + SeekableStreamSupervisorIOConfig ioConfig + ) + { + return new SeekableStreamIndexTaskIOConfig( + groupId, + baseSequenceName, + new SeekableStreamStartSequenceNumbers<>(STREAM, startPartitions, exclusiveStartSequenceNumberPartitions), + new SeekableStreamEndSequenceNumbers<>(STREAM, endPartitions), + true, + minimumMessageTime, + maximumMessageTime, + ioConfig.getInputFormat() + ) + { + }; + } + + @Override + protected List> createIndexTasks( + int replicas, + String baseSequenceName, + ObjectMapper sortingMapper, + TreeMap> sequenceOffsets, + SeekableStreamIndexTaskIOConfig taskIoConfig, + SeekableStreamIndexTaskTuningConfig taskTuningConfig, + RowIngestionMetersFactory rowIngestionMetersFactory + ) + { + return null; + } + + @Override + protected int getTaskGroupIdForPartition(String partition) + { + return 0; + } + + @Override + protected boolean checkSourceMetadataMatch(DataSourceMetadata metadata) + { + return true; + } + + @Override + protected boolean doesTaskTypeMatchSupervisor(Task task) + { + return true; + } + + @Override + protected SeekableStreamDataSourceMetadata createDataSourceMetaDataForReset( + String stream, + Map map + ) + { + return null; + } + + @Override + protected OrderedSequenceNumber makeSequenceNumber(String seq, boolean isExclusive) + { + return new OrderedSequenceNumber(seq, isExclusive) + { + @Override + public int compareTo(OrderedSequenceNumber o) + { + return new BigInteger(this.get()).compareTo(new BigInteger(o.get())); + } + }; + } + + @Override + protected Map getRecordLagPerPartition(Map currentOffsets) + { + return null; + } + + @Override + protected Map getTimeLagPerPartition(Map currentOffsets) + { + return null; + } + + @Override + protected RecordSupplier setupRecordSupplier() + { + return recordSupplier; + } + + @Override + protected SeekableStreamSupervisorReportPayload createReportPayload( + int numPartitions, + boolean includeOffsets + ) + { + return new SeekableStreamSupervisorReportPayload( + DATASOURCE, + STREAM, + 1, + 1, + 1L, + null, + null, + null, + null, + null, + null, + false, + true, + null, + null, + null + ) + { + }; + } + + @Override + protected String getNotSetMarker() + { + return "NOT_SET"; + } + + @Override + protected String getEndOfPartitionMarker() + { + return "EOF"; + } + + @Override + protected boolean isEndOfShard(String seqNum) + { + return false; + } + + @Override + protected boolean isShardExpirationMarker(String seqNum) + { + return false; + } + + @Override + protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() + { + return false; + } + } + + private class TestSeekableStreamSupervisor extends BaseTestSeekableStreamSupervisor + { + @Override + protected void scheduleReporting(ScheduledExecutorService reportingExec) + { + // do nothing + } + + @Override + public void collectLag(ArrayList lags) + { + } + } + + + private class TesstSeekableStreamSupervisorSpec extends SeekableStreamSupervisorSpec + { + private SeekableStreamSupervisor supervisor; + private String id; + + public TesstSeekableStreamSupervisorSpec(SeekableStreamSupervisorIngestionSpec ingestionSchema, + @Nullable Map context, + Boolean suspended, + TaskStorage taskStorage, + TaskMaster taskMaster, + IndexerMetadataStorageCoordinator indexerMetadataStorageCoordinator, + SeekableStreamIndexTaskClientFactory indexTaskClientFactory, + ObjectMapper mapper, + ServiceEmitter emitter, + DruidMonitorSchedulerConfig monitorSchedulerConfig, + RowIngestionMetersFactory rowIngestionMetersFactory, + SupervisorStateManagerConfig supervisorStateManagerConfig, + SeekableStreamSupervisor supervisor, + String id) + { + super( + ingestionSchema, + context, + suspended, + taskStorage, + taskMaster, + indexerMetadataStorageCoordinator, + indexTaskClientFactory, + mapper, + emitter, + monitorSchedulerConfig, + rowIngestionMetersFactory, + supervisorStateManagerConfig); + + this.supervisor = supervisor; + this.id = id; + } + + @Override + public List getDataSources() + { + return new ArrayList<>(); + } + + @Override + public String getId() + { + return id; + } + + @Override + public Supervisor createSupervisor() + { + return supervisor; + } + + @Override + public String getType() + { + return null; + } + + @Override + public String getSource() + { + return null; + } + + @Override + protected SeekableStreamSupervisorSpec toggleSuspend(boolean suspend) + { + return null; + } + } + + private static SeekableStreamSupervisorTuningConfig getTuningConfig() + { + return new SeekableStreamSupervisorTuningConfig() + { + @Override + public Integer getWorkerThreads() + { + return 1; + } + + @Override + public Integer getChatThreads() + { + return 1; + } + + @Override + public Long getChatRetries() + { + return 1L; + } + + @Override + public Duration getHttpTimeout() + { + return new Period("PT1M").toStandardDuration(); + } + + @Override + public Duration getShutdownTimeout() + { + return new Period("PT1S").toStandardDuration(); + } + + @Override + public Duration getRepartitionTransitionDuration() + { + return new Period("PT2M").toStandardDuration(); + } + + @Override + public Duration getOffsetFetchPeriod() + { + return new Period("PT5M").toStandardDuration(); + } + + @Override + public SeekableStreamIndexTaskTuningConfig convertToTaskTuningConfig() + { + return new SeekableStreamIndexTaskTuningConfig( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ) + { + @Override + public SeekableStreamIndexTaskTuningConfig withBasePersistDirectory(File dir) + { + return null; + } + + @Override + public String toString() + { + return null; + } + }; + } + }; + } + + @Test + public void testAutoScalerCreated() + { + HashMap dynamicAllocationTasksProperties = new HashMap<>(); + dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", true); + dynamicAllocationTasksProperties.put("metricsCollectionIntervalMillis", 500); + dynamicAllocationTasksProperties.put("metricsCollectionRangeMillis", 500); + dynamicAllocationTasksProperties.put("scaleOutThreshold", 5000000); + dynamicAllocationTasksProperties.put("triggerScaleOutThresholdFrequency", 0.3); + dynamicAllocationTasksProperties.put("scaleInThreshold", 1000000); + dynamicAllocationTasksProperties.put("triggerScaleInThresholdFrequency", 0.8); + dynamicAllocationTasksProperties.put("dynamicCheckStartDelayMillis", 0); + dynamicAllocationTasksProperties.put("dynamicCheckPeriod", 100); + dynamicAllocationTasksProperties.put("taskCountMax", 8); + dynamicAllocationTasksProperties.put("taskCountMin", 1); + dynamicAllocationTasksProperties.put("scaleInStep", 1); + dynamicAllocationTasksProperties.put("scaleOutStep", 2); + dynamicAllocationTasksProperties.put("minTriggerDynamicFrequencyMillis", 1200000); + + EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes(); + EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes(); + EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); + EasyMock.replay(ingestionSchema); + + EasyMock.expect(seekableStreamSupervisorIOConfig.getDynamicAllocationTasksProperties()).andReturn(dynamicAllocationTasksProperties).anyTimes(); + EasyMock.replay(seekableStreamSupervisorIOConfig); + + EasyMock.expect(supervisor4.getSupervisorTaskInfos()).andReturn(new HashMap()).anyTimes(); + EasyMock.replay(supervisor4); + + TesstSeekableStreamSupervisorSpec spec = new TesstSeekableStreamSupervisorSpec(ingestionSchema, + null, + false, + taskStorage, + taskMaster, + indexerMetadataStorageCoordinator, + indexTaskClientFactory, + mapper, + emitter, + monitorSchedulerConfig, + rowIngestionMetersFactory, + supervisorStateManagerConfig, + supervisor4, + "id1"); + SupervisorTaskAutoscaler autoscaler = spec.createAutoscaler(supervisor4); + Assert.assertTrue(autoscaler instanceof DefaultAutoScaler); + + dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", false); + SupervisorTaskAutoscaler autoscaler2 = spec.createAutoscaler(supervisor4); + Assert.assertTrue(autoscaler2 instanceof DummyAutoScaler); + + dynamicAllocationTasksProperties.remove("enableDynamicAllocationTasks"); + SupervisorTaskAutoscaler autoscaler3 = spec.createAutoscaler(supervisor4); + Assert.assertTrue(autoscaler3 instanceof DummyAutoScaler); + + } + + @Test + public void testSeekableStreamSupervisorSpecWithScaleOut() throws InterruptedException + { + + EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes(); + + EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes(); + EasyMock.expect(spec.getIoConfig()).andReturn(getIOConfig(1, true)).anyTimes(); + EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes(); + EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes(); + EasyMock.expect(spec.isSuspended()).andReturn(false).anyTimes(); + EasyMock.replay(spec); + + EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes(); + EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes(); + EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); + EasyMock.replay(ingestionSchema); + + EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes(); + EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes(); + EasyMock.replay(taskMaster); + + TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); + DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, getScaleOutProperties(), spec); + supervisor.start(); + autoScaler.start(); + supervisor.runInternal(); + int taskCountBeforeScaleOut = supervisor.getIoConfig().getTaskCount(); + Assert.assertEquals(1, taskCountBeforeScaleOut); + Thread.sleep(1 * 1000); + int taskCountAfterScaleOut = supervisor.getIoConfig().getTaskCount(); + Assert.assertEquals(2, taskCountAfterScaleOut); + + autoScaler.reset(); + autoScaler.stop(); + + } + + @Test + public void testSeekableStreamSupervisorSpecWithScaleIn() throws InterruptedException + { + + EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes(); + + EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes(); + EasyMock.expect(spec.getIoConfig()).andReturn(getIOConfig(2, false)).anyTimes(); + EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes(); + EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes(); + EasyMock.expect(spec.isSuspended()).andReturn(false).anyTimes(); + EasyMock.replay(spec); + + EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes(); + EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes(); + EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); + EasyMock.replay(ingestionSchema); + + EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes(); + EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes(); + EasyMock.replay(taskMaster); + + TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); + DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, getScaleInProperties(), spec); + supervisor.start(); + autoScaler.start(); + supervisor.runInternal(); + int taskCountBeforeScaleOut = supervisor.getIoConfig().getTaskCount(); + Assert.assertEquals(2, taskCountBeforeScaleOut); + Thread.sleep(1 * 1000); + int taskCountAfterScaleOut = supervisor.getIoConfig().getTaskCount(); + Assert.assertEquals(1, taskCountAfterScaleOut); + + autoScaler.reset(); + autoScaler.stop(); + } + + @Test + public void testSeekableStreamSupervisorSpecWithScaleDisable() throws InterruptedException + { + + SeekableStreamSupervisorIOConfig seekableStreamSupervisorIOConfig = new SeekableStreamSupervisorIOConfig( + "stream", + new JsonInputFormat(new JSONPathSpec(true, ImmutableList.of()), ImmutableMap.of(), false), + 1, + 1, + new Period("PT1H"), + new Period("P1D"), + new Period("PT30S"), + false, + new Period("PT30M"), + null, + null, null, null + ) {}; + EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes(); + + EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes(); + EasyMock.expect(spec.getIoConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes(); + EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes(); + EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes(); + EasyMock.expect(spec.isSuspended()).andReturn(false).anyTimes(); + EasyMock.replay(spec); + + EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(this.seekableStreamSupervisorIOConfig).anyTimes(); + EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes(); + EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); + EasyMock.replay(ingestionSchema); + + EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes(); + EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes(); + EasyMock.replay(taskMaster); + + TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); + DummyAutoScaler autoScaler = new DummyAutoScaler(supervisor, DATASOURCE); + supervisor.start(); + autoScaler.start(); + supervisor.runInternal(); + int taskCountBeforeScaleOut = supervisor.getIoConfig().getTaskCount(); + Assert.assertEquals(1, taskCountBeforeScaleOut); + Thread.sleep(1 * 1000); + int taskCountAfterScaleOut = supervisor.getIoConfig().getTaskCount(); + Assert.assertEquals(1, taskCountAfterScaleOut); + + autoScaler.reset(); + autoScaler.stop(); + } + + private static DataSchema getDataSchema() + { + List dimensions = new ArrayList<>(); + dimensions.add(StringDimensionSchema.create("dim1")); + dimensions.add(StringDimensionSchema.create("dim2")); + + return new DataSchema( + DATASOURCE, + new TimestampSpec("timestamp", "iso", null), + new DimensionsSpec( + dimensions, + null, + null + ), + new AggregatorFactory[]{new CountAggregatorFactory("rows")}, + new UniformGranularitySpec( + Granularities.HOUR, + Granularities.NONE, + ImmutableList.of() + ), + null + ); + } + + private static SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boolean scaleOut) + { + if (scaleOut) { + return new SeekableStreamSupervisorIOConfig( + "stream", + new JsonInputFormat(new JSONPathSpec(true, ImmutableList.of()), ImmutableMap.of(), false), + 1, + taskCount, + new Period("PT1H"), + new Period("P1D"), + new Period("PT30S"), + false, + new Period("PT30M"), + null, + null, getScaleOutProperties(), null + ) {}; + } else { + return new SeekableStreamSupervisorIOConfig( + "stream", + new JsonInputFormat(new JSONPathSpec(true, ImmutableList.of()), ImmutableMap.of(), false), + 1, + taskCount, + new Period("PT1H"), + new Period("P1D"), + new Period("PT30S"), + false, + new Period("PT30M"), + null, + null, getScaleInProperties(), null + ) {}; + } + } + + private static Map getScaleOutProperties() + { + HashMap dynamicAllocationTasksProperties = new HashMap<>(); + dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", true); + dynamicAllocationTasksProperties.put("metricsCollectionIntervalMillis", 500); + dynamicAllocationTasksProperties.put("metricsCollectionRangeMillis", 500); + dynamicAllocationTasksProperties.put("scaleOutThreshold", 0); + dynamicAllocationTasksProperties.put("triggerScaleOutThresholdFrequency", 0.0); + dynamicAllocationTasksProperties.put("scaleInThreshold", 1000000); + dynamicAllocationTasksProperties.put("triggerScaleInThresholdFrequency", 0.8); + dynamicAllocationTasksProperties.put("dynamicCheckStartDelayMillis", 0); + dynamicAllocationTasksProperties.put("dynamicCheckPeriod", 100); + dynamicAllocationTasksProperties.put("taskCountMax", 2); + dynamicAllocationTasksProperties.put("taskCountMin", 1); + dynamicAllocationTasksProperties.put("scaleInStep", 1); + dynamicAllocationTasksProperties.put("scaleOutStep", 2); + dynamicAllocationTasksProperties.put("minTriggerDynamicFrequencyMillis", 1200000); + return dynamicAllocationTasksProperties; + } + + private static Map getScaleInProperties() + { + HashMap dynamicAllocationTasksProperties = new HashMap<>(); + dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", true); + dynamicAllocationTasksProperties.put("metricsCollectionIntervalMillis", 500); + dynamicAllocationTasksProperties.put("metricsCollectionRangeMillis", 500); + dynamicAllocationTasksProperties.put("scaleOutThreshold", 8000000); + dynamicAllocationTasksProperties.put("triggerScaleOutThresholdFrequency", 0.3); + dynamicAllocationTasksProperties.put("scaleInThreshold", 0); + dynamicAllocationTasksProperties.put("triggerScaleInThresholdFrequency", 0.0); + dynamicAllocationTasksProperties.put("dynamicCheckStartDelayMillis", 0); + dynamicAllocationTasksProperties.put("dynamicCheckPeriod", 100); + dynamicAllocationTasksProperties.put("taskCountMax", 2); + dynamicAllocationTasksProperties.put("taskCountMin", 1); + dynamicAllocationTasksProperties.put("scaleInStep", 1); + dynamicAllocationTasksProperties.put("scaleOutStep", 2); + dynamicAllocationTasksProperties.put("minTriggerDynamicFrequencyMillis", 1200000); + return dynamicAllocationTasksProperties; + } + +} From 172cff7904038e2b700ca427cad490ea7751cf74 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Sun, 17 Jan 2021 13:39:56 +0800 Subject: [PATCH 20/47] add more uts --- .../MaterializedViewSupervisorSpecTest.java | 77 +++++++++++++++++++ indexing-service/pom.xml | 6 +- .../SeekableStreamSupervisorSpec.java | 4 +- .../autoscaler/DefaultAutoScaler.java | 3 +- .../autoscaler/DummyAutoScaler.java | 3 +- .../OverlordSecurityResourceFilterTest.java | 2 +- .../SeekableStreamSupervisorSpecTest.java | 6 +- .../indexing/NoopSupervisorSpecTest.java | 64 +++++++++++++++ .../SQLMetadataSupervisorManagerTest.java | 3 +- .../druid/metadata/TestSupervisorSpec.java | 3 +- 10 files changed, 157 insertions(+), 14 deletions(-) rename {server/src/main/java/org/apache/druid/indexing/overlord => indexing-service/src/main/java/org/apache/druid/indexing/seekablestream}/supervisor/autoscaler/DefaultAutoScaler.java (98%) rename {server/src/main/java/org/apache/druid/indexing/overlord => indexing-service/src/main/java/org/apache/druid/indexing/seekablestream}/supervisor/autoscaler/DummyAutoScaler.java (88%) create mode 100644 server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java diff --git a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java index c82b8b8fae6a..a861ad197e84 100644 --- a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java +++ b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java @@ -29,7 +29,9 @@ import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.TaskMaster; import org.apache.druid.indexing.overlord.TaskStorage; +import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.metadata.MetadataSupervisorManager; import org.apache.druid.metadata.SqlSegmentsMetadataManager; @@ -50,6 +52,9 @@ import org.junit.rules.ExpectedException; import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; +import java.util.concurrent.Callable; public class MaterializedViewSupervisorSpecTest { @@ -155,6 +160,78 @@ public void testSupervisorSerialization() throws IOException Assert.assertEquals(expected.getMetrics(), spec.getMetrics()); } + @Test + public void testMaterializedViewSupervisorSpecCreated() + { + Exception ex = null; + + try { + MaterializedViewSupervisorSpec spec = new MaterializedViewSupervisorSpec( + "wikiticker", + new DimensionsSpec( + Lists.newArrayList( + new StringDimensionSchema("isUnpatrolled"), + new StringDimensionSchema("metroCode"), + new StringDimensionSchema("namespace"), + new StringDimensionSchema("page"), + new StringDimensionSchema("regionIsoCode"), + new StringDimensionSchema("regionName"), + new StringDimensionSchema("user") + ), + null, + null + ), + new AggregatorFactory[]{ + new CountAggregatorFactory("count"), + new LongSumAggregatorFactory("added", "added") + }, + HadoopTuningConfig.makeDefaultTuningConfig(), + null, + null, + null, + null, + null, + false, + objectMapper, + null, + null, + null, + null, + null, + new MaterializedViewTaskConfig(), + EasyMock.createMock(AuthorizerMapper.class), + new NoopChatHandlerProvider(), + new SupervisorStateManagerConfig() + ); + Supervisor supervisor = spec.createSupervisor(); + Assert.assertTrue(supervisor instanceof MaterializedViewSupervisor); + + SupervisorTaskAutoscaler autoscaler = spec.createAutoscaler(supervisor); + Assert.assertNull(autoscaler); + + supervisor.collectLag(new ArrayList<>()); + + Map supervisorTaskInfos = supervisor.getSupervisorTaskInfos(); + Assert.assertNull(supervisorTaskInfos); + + Callable noop = new Callable() { + @Override + public Integer call() + { + return -1; + } + }; + Runnable runnable = supervisor.buildDynamicAllocationTask(noop); + Assert.assertNull(runnable); + + } + catch (Exception e) { + ex = e; + } + + Assert.assertNull(ex); + } + @Test public void testSuspendResuume() throws IOException { diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 33f4df5d043e..5e7c4a6cab1b 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -62,7 +62,11 @@ druid-hll ${project.parent.version} - + + org.apache.commons + commons-collections4 + 4.2 + io.dropwizard.metrics metrics-core diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java index 6e08aa0e0298..d26caddf52b6 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java @@ -33,10 +33,10 @@ import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.DefaultAutoScaler; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.DummyAutoScaler; import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientFactory; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScaler; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DummyAutoScaler; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DefaultAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java similarity index 98% rename from server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DefaultAutoScaler.java rename to indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java index 0ebd86c360ed..abcb9360a586 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DefaultAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java @@ -17,11 +17,12 @@ * under the License. */ -package org.apache.druid.indexing.overlord.supervisor.autoscaler; +package org.apache.druid.indexing.seekablestream.supervisor.autoscaler; import org.apache.commons.collections4.queue.CircularFifoQueue; import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.emitter.EmittingLogger; diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DummyAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DummyAutoScaler.java similarity index 88% rename from server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DummyAutoScaler.java rename to indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DummyAutoScaler.java index 8f13e66a93b6..2d779e09c33e 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/DummyAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DummyAutoScaler.java @@ -17,9 +17,10 @@ * under the License. */ -package org.apache.druid.indexing.overlord.supervisor.autoscaler; +package org.apache.druid.indexing.seekablestream.supervisor.autoscaler; import org.apache.druid.indexing.overlord.supervisor.Supervisor; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; public class DummyAutoScaler implements SupervisorTaskAutoscaler { diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java index 4b831a9607e1..062908b17085 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java @@ -32,8 +32,8 @@ import org.apache.druid.indexing.overlord.supervisor.SupervisorManager; import org.apache.druid.indexing.overlord.supervisor.SupervisorResource; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.DummyAutoScaler; import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DummyAutoScaler; import org.apache.druid.indexing.worker.http.WorkerResource; import org.apache.druid.server.http.security.ResourceFilterTestHelper; import org.apache.druid.server.security.AuthorizerMapper; diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 37611a13c1b8..e5ffa67ba755 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -35,10 +35,7 @@ import org.apache.druid.indexing.overlord.TaskMaster; import org.apache.druid.indexing.overlord.TaskStorage; import org.apache.druid.indexing.overlord.supervisor.Supervisor; -import org.apache.druid.indexing.overlord.supervisor.SupervisorManager; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.DefaultAutoScaler; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.DummyAutoScaler; import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.indexing.seekablestream.common.OrderedSequenceNumber; import org.apache.druid.indexing.seekablestream.common.RecordSupplier; @@ -48,6 +45,8 @@ import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorReportPayload; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorSpec; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorTuningConfig; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScaler; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DummyAutoScaler; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.parsers.JSONPathSpec; import org.apache.druid.java.util.emitter.service.ServiceEmitter; @@ -60,7 +59,6 @@ import org.apache.druid.server.metrics.DruidMonitorSchedulerConfig; import org.easymock.EasyMock; import org.easymock.EasyMockSupport; -import org.easymock.Mock; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.Period; diff --git a/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java b/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java new file mode 100644 index 000000000000..9d5eb6b27fbe --- /dev/null +++ b/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing; + +import org.apache.druid.indexing.overlord.supervisor.NoopSupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.Supervisor; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.Callable; + +public class NoopSupervisorSpecTest +{ + @Test + public void testNoopSupervisorSpecWithAutoscaler() + { + Exception e = null; + try { + NoopSupervisorSpec noopSupervisorSpec = new NoopSupervisorSpec(null, Collections.singletonList("datasource1")); + Supervisor supervisor = noopSupervisorSpec.createSupervisor(); + SupervisorTaskAutoscaler autoscaler = noopSupervisorSpec.createAutoscaler(supervisor); + Assert.assertNull(autoscaler); + Callable noop = new Callable() { + @Override + public Integer call() + { + return -1; + } + }; + Runnable runnable = supervisor.buildDynamicAllocationTask(noop); + Assert.assertNull(runnable); + + Map supervisorTaskInfos = supervisor.getSupervisorTaskInfos(); + Assert.assertNull(supervisorTaskInfos); + + supervisor.collectLag(new ArrayList()); + } + catch (Exception ex) { + e = ex; + } + Assert.assertNull(e); + } +} diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java index 624157a52552..89f8b6a05ad4 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java @@ -25,7 +25,6 @@ import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; import org.apache.druid.indexing.overlord.supervisor.VersionedSupervisorSpec; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.DummyAutoScaler; import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.StringUtils; @@ -190,7 +189,7 @@ public Supervisor createSupervisor() @Override public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) { - return new DummyAutoScaler(supervisor, null); + return null; } @Override diff --git a/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java b/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java index 407520a0968d..9eb3b5435ad9 100644 --- a/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java +++ b/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.DummyAutoScaler; import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import java.util.List; @@ -57,7 +56,7 @@ public Supervisor createSupervisor() @Override public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) { - return new DummyAutoScaler(supervisor, null); + return null; } @Override From 57811be4b6fb66a4cfe7e32267a5730bb7e57e48 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Sun, 17 Jan 2021 14:49:13 +0800 Subject: [PATCH 21/47] fix inner class check --- .../seekablestream/SeekableStreamSupervisorSpecTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index e5ffa67ba755..69a50c282adf 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -339,7 +339,7 @@ public void collectLag(ArrayList lags) } - private class TesstSeekableStreamSupervisorSpec extends SeekableStreamSupervisorSpec + private static class TesstSeekableStreamSupervisorSpec extends SeekableStreamSupervisorSpec { private SeekableStreamSupervisor supervisor; private String id; From ff8105c173082116a9d346c6cd3caae44fda42d0 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 26 Jan 2021 10:39:38 +0800 Subject: [PATCH 22/47] add IT for kafka ingestion with autoscaler --- .../indexer/AbstractStreamIndexingTest.java | 65 +++++++++++++++++ ...rviceNonTransactionalParallelizedTest.java | 10 +++ ...ervisor_with_autoscaler_spec_template.json | 73 +++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java index 5c2a18edbac1..4d9648842fcb 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java @@ -74,10 +74,14 @@ public abstract class AbstractStreamIndexingTest extends AbstractIndexerTest private static final String QUERIES_FILE = "/stream/queries/stream_index_queries.json"; private static final String SUPERVISOR_SPEC_TEMPLATE_FILE = "supervisor_spec_template.json"; + private static final String SUPERVISOR_WITH_AUTOSCALER_SPEC_TEMPLATE_FILE = "supervisor_with_autoscaler_spec_template.json"; protected static final String DATA_RESOURCE_ROOT = "/stream/data"; protected static final String SUPERVISOR_SPEC_TEMPLATE_PATH = String.join("/", DATA_RESOURCE_ROOT, SUPERVISOR_SPEC_TEMPLATE_FILE); + protected static final String SUPERVISOR_WITH_AUTOSCALER_SPEC_TEMPLATE_PATH = + String.join("/", DATA_RESOURCE_ROOT, SUPERVISOR_WITH_AUTOSCALER_SPEC_TEMPLATE_FILE); + protected static final String SERIALIZER_SPEC_DIR = "serializer"; protected static final String INPUT_FORMAT_SPEC_DIR = "input_format"; protected static final String INPUT_ROW_PARSER_SPEC_DIR = "parser"; @@ -294,6 +298,67 @@ protected void doTestIndexDataWithStartStopSupervisor(@Nullable Boolean transact } } + protected void doTestIndexDataWithAutoscaler(@Nullable Boolean transactionEnabled) throws Exception + { + final GeneratedTestConfig generatedTestConfig = new GeneratedTestConfig( + INPUT_FORMAT, + getResourceAsString(JSON_INPUT_FORMAT_PATH) + ); + try ( + final Closeable closer = createResourceCloser(generatedTestConfig); + final StreamEventWriter streamEventWriter = createStreamEventWriter(config, transactionEnabled) + ) { + final String taskSpec = generatedTestConfig.getStreamIngestionPropsTransform() + .apply(getResourceAsString(SUPERVISOR_WITH_AUTOSCALER_SPEC_TEMPLATE_PATH)); + LOG.info("supervisorSpec: [%s]\n", taskSpec); + // Start supervisor + generatedTestConfig.setSupervisorId(indexer.submitSupervisor(taskSpec)); + LOG.info("Submitted supervisor"); + // Start generating half of the data + int secondsToGenerateRemaining = TOTAL_NUMBER_OF_SECOND; + int secondsToGenerateFirstRound = TOTAL_NUMBER_OF_SECOND / 2; + secondsToGenerateRemaining = secondsToGenerateRemaining - secondsToGenerateFirstRound; + final StreamGenerator streamGenerator = new WikipediaStreamEventStreamGenerator( + new JsonEventSerializer(jsonMapper), + EVENTS_PER_SECOND, + CYCLE_PADDING_MS + ); + long numWritten = streamGenerator.run( + generatedTestConfig.getStreamName(), + streamEventWriter, + secondsToGenerateFirstRound, + FIRST_EVENT_TIME + ); + // Verify supervisor is healthy before suspension + ITRetryUtil.retryUntil( + () -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), + true, + 10000, + 30, + "Waiting for supervisor to be healthy" + ); + // Start generating remainning half of the data + numWritten += streamGenerator.run( + generatedTestConfig.getStreamName(), + streamEventWriter, + secondsToGenerateRemaining, + FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound) + ); + // wait for autoScaling task numbers from 1 to 2. + ITRetryUtil.retryUntil( + () -> indexer.getRunningTasks().size() == 2, + true, + 10000, + 50, + "Waiting for supervisor to be healthy" + ); + // Verify that supervisor can catch up with the stream + verifyIngestedData(generatedTestConfig, numWritten); + } + } + + + protected void doTestIndexDataWithStreamReshardSplit(@Nullable Boolean transactionEnabled) throws Exception { // Reshard the stream from STREAM_SHARD_COUNT to STREAM_SHARD_COUNT * 2 diff --git a/integration-tests/src/test/java/org/apache/druid/tests/parallelized/ITKafkaIndexingServiceNonTransactionalParallelizedTest.java b/integration-tests/src/test/java/org/apache/druid/tests/parallelized/ITKafkaIndexingServiceNonTransactionalParallelizedTest.java index 2c648ea630b2..967ff524b2ea 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/parallelized/ITKafkaIndexingServiceNonTransactionalParallelizedTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/parallelized/ITKafkaIndexingServiceNonTransactionalParallelizedTest.java @@ -52,6 +52,16 @@ public void testKafkaIndexDataWithStartStopSupervisor() throws Exception doTestIndexDataWithStartStopSupervisor(false); } + /** + * This test can be run concurrently with other tests as it creates/modifies/teardowns a unique datasource + * and supervisor maintained and scoped within this test only + */ + @Test + public void testKafkaIndexDataWithWithAutoscaler() throws Exception + { + doTestIndexDataWithAutoscaler(false); + } + /** * This test can be run concurrently with other tests as it creates/modifies/teardowns a unique datasource * and supervisor maintained and scoped within this test only diff --git a/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json b/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json new file mode 100644 index 000000000000..7f6c50817b62 --- /dev/null +++ b/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json @@ -0,0 +1,73 @@ +{ + "type": "%%STREAM_TYPE%%", + "dataSchema": { + "dataSource": "%%DATASOURCE%%", + "parser": %%PARSER%%, + "timestampSpec": { + "column": "timestamp", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": ["page", "language", "user", "unpatrolled", "newPage", "robot", "anonymous", "namespace", "continent", "country", "region", "city"], + "dimensionExclusions": [], + "spatialDimensions": [] + }, + "metricsSpec": [ + { + "type": "count", + "name": "count" + }, + { + "type": "doubleSum", + "name": "added", + "fieldName": "added" + }, + { + "type": "doubleSum", + "name": "deleted", + "fieldName": "deleted" + }, + { + "type": "doubleSum", + "name": "delta", + "fieldName": "delta" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "MINUTE", + "queryGranularity": "NONE" + } + }, + "tuningConfig": { + "type": "%%STREAM_TYPE%%", + "intermediatePersistPeriod": "PT30S", + "maxRowsPerSegment": 5000000, + "maxRowsInMemory": 500000 + }, + "ioConfig": { + "%%TOPIC_KEY%%": "%%TOPIC_VALUE%%", + "%%STREAM_PROPERTIES_KEY%%": %%STREAM_PROPERTIES_VALUE%%, + "dynamicAllocationTasksProperties": { + "enableDynamicAllocationTasks": true, + "metricsCollectionIntervalMillis": 500, + "metricsCollectionRangeMillis": 500, + "scaleOutThreshold": 0, + "triggerScaleOutThresholdFrequency": 0.0, + "scaleInThreshold": 1000000, + "triggerScaleInThresholdFrequency": 0.9, + "dynamicCheckStartDelayMillis": 0, + "dynamicCheckPeriod": 100, + "taskCountMax": 2, + "taskCountMin": 1, + "scaleInStep": 1, + "scaleOutStep": 2, + "minTriggerDynamicFrequencyMillis": 600000 + }, + "taskCount": 1, + "replicas": 1, + "taskDuration": "PT30S", + "%%USE_EARLIEST_KEY%%": true, + "inputFormat" : %%INPUT_FORMAT%% + } +} From 05571f7e0a124f3971ecc0716d85d3fb30581d5b Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 26 Jan 2021 14:11:02 +0800 Subject: [PATCH 23/47] add new IT in groups=kafka-index named testKafkaIndexDataWithWithAutoscaler --- .../indexer/AbstractStreamIndexingTest.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java index 4d9648842fcb..c38b67ae4286 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/indexer/AbstractStreamIndexingTest.java @@ -331,27 +331,30 @@ protected void doTestIndexDataWithAutoscaler(@Nullable Boolean transactionEnable ); // Verify supervisor is healthy before suspension ITRetryUtil.retryUntil( - () -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), + () -> SupervisorStateManager.BasicState.RUNNING.equals(indexer.getSupervisorStatus(generatedTestConfig.getSupervisorId())), true, 10000, 30, "Waiting for supervisor to be healthy" ); - // Start generating remainning half of the data - numWritten += streamGenerator.run( - generatedTestConfig.getStreamName(), - streamEventWriter, - secondsToGenerateRemaining, - FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound) - ); + // wait for autoScaling task numbers from 1 to 2. ITRetryUtil.retryUntil( () -> indexer.getRunningTasks().size() == 2, true, 10000, 50, - "Waiting for supervisor to be healthy" + "waiting for autoScaling task numbers from 1 to 2" ); + + // Start generating remainning half of the data + numWritten += streamGenerator.run( + generatedTestConfig.getStreamName(), + streamEventWriter, + secondsToGenerateRemaining, + FIRST_EVENT_TIME.plusSeconds(secondsToGenerateFirstRound) + ); + // Verify that supervisor can catch up with the stream verifyIngestedData(generatedTestConfig, numWritten); } From 87a694ad10daa29b8934599a05e35bdbfea14059 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 2 Feb 2021 15:28:01 +0800 Subject: [PATCH 24/47] review change --- .idea/misc.xml | 4 +- .../extensions-core/kafka-ingestion.md | 4 +- .../MaterializedViewSupervisor.java | 15 +-- .../MaterializedViewSupervisorSpec.java | 7 -- .../MaterializedViewSupervisorSpecTest.java | 19 ++-- .../kafka/supervisor/KafkaSupervisor.java | 8 +- .../supervisor/KafkaSupervisorIOConfig.java | 4 +- .../kafka/supervisor/KafkaSupervisorTest.java | 33 +++--- .../kinesis/supervisor/KinesisSupervisor.java | 4 +- .../supervisor/KinesisSupervisorIOConfig.java | 11 +- .../supervisor/SeekableStreamSupervisor.java | 50 ++++---- .../SeekableStreamSupervisorIOConfig.java | 10 +- .../SeekableStreamSupervisorSpec.java | 15 ++- .../autoscaler/DefaultAutoScaler.java | 57 ++++++---- .../SeekableStreamSupervisorSpecTest.java | 107 +++++++++--------- .../SeekableStreamSupervisorStateTest.java | 39 ++++--- ...ervisor_with_autoscaler_spec_template.json | 4 +- .../supervisor/NoopSupervisorSpec.java | 20 +--- .../overlord/supervisor/Supervisor.java | 15 +-- .../overlord/supervisor/SupervisorSpec.java | 5 +- .../supervisor/autoscaler/LagStats.java | 49 ++++++++ .../indexing/NoopSupervisorSpecTest.java | 10 +- 22 files changed, 272 insertions(+), 218 deletions(-) create mode 100644 server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/LagStats.java diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 5c3514ce18d5..28cdd807c98a 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -146,13 +146,13 @@ A sample supervisor spec is shown below: |`lateMessageRejectionStartDateTime`|ISO8601 DateTime|Configure tasks to reject messages with timestamps earlier than this date time; for example if this is set to `2016-01-01T11:00Z` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline).|no (default == none)| |`lateMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps earlier than this period before the task was created; for example if this is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline). Please note that only one of `lateMessageRejectionPeriod` or `lateMessageRejectionStartDateTime` can be specified.|no (default == none)| |`earlyMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps later than this period after the task reached its taskDuration; for example if this is set to `PT1H`, the taskDuration is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps later than *2016-01-01T14:00Z* will be dropped. **Note:** Tasks sometimes run past their task duration, for example, in cases of supervisor failover. Setting earlyMessageRejectionPeriod too low may cause messages to be dropped unexpectedly whenever a task runs past its originally configured task duration.|no (default == none)| -|`dynamicAllocationTasksProperties`|Object|`dynamicAllocationTasksProperties` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. See [Dynamic Allocation Tasks Properties](#Dynamic Allocation Tasks Properties) for details.|no (default == null)| +|`autoscalerConfig`|Object|`autoscalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for kafka indexing as of now. See [Dynamic Allocation Tasks Properties](#Dynamic Allocation Tasks Properties) for details.|no (default == null)| #### Dynamic Allocation Tasks Properties | Property | Description | Default | | ------------- | ------------- | ------------- | -| `enableDynamicAllocationTasks` | whether enable this feature or not | false | +| `enableTaskAutoscaler` | whether enable this feature or not. Set false here will disable autoscaler even though autoscalerConfig is not null| true | | `metricsCollectionIntervalMillis` | Define the frequency of lag points collection. | 30000 | | `metricsCollectionRangeMillis` | The total time window of lag collection, Use with `metricsCollectionIntervalMillis`，it means that in the recent `metricsCollectionRangeMill`, collect lag metric points every `metricsCollectionIntervalMillis`. | 600000 | | `scaleOutThreshold` | The Threshold of scale out action | 6000000 | diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java index be6289d2cb00..f456cb611c9b 100644 --- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java +++ b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java @@ -37,6 +37,7 @@ import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorReport; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.JodaUtils; @@ -62,7 +63,6 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; -import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; public class MaterializedViewSupervisor implements Supervisor @@ -284,20 +284,15 @@ public void checkpoint(int taskGroupId, DataSourceMetadata checkpointMetadata) } @Override - public void collectLag(ArrayList lags) + public LagStats computeLagStats() { + throw new UnsupportedOperationException("Compute Lag Stats not supported in MaterializedViewSupervisor"); } @Override - public Runnable buildDynamicAllocationTask(Callable scaleAction) + public int getActiveTaskGroupsCount() { - return null; - } - - @Override - public Map getSupervisorTaskInfos() - { - return null; + throw new UnsupportedOperationException("Get Active Task Groups Count is not supported in MaterializedViewSupervisor"); } /** diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java index c46a22098b7e..db63a7316d1f 100644 --- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java +++ b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java @@ -39,7 +39,6 @@ import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; @@ -362,12 +361,6 @@ public Supervisor createSupervisor() ); } - @Override - public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) - { - return null; - } - @Override public List getDataSources() { diff --git a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java index a861ad197e84..da6c5a3d959d 100644 --- a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java +++ b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java @@ -52,8 +52,6 @@ import org.junit.rules.ExpectedException; import java.io.IOException; -import java.util.ArrayList; -import java.util.Map; import java.util.concurrent.Callable; public class MaterializedViewSupervisorSpecTest @@ -209,10 +207,19 @@ public void testMaterializedViewSupervisorSpecCreated() SupervisorTaskAutoscaler autoscaler = spec.createAutoscaler(supervisor); Assert.assertNull(autoscaler); - supervisor.collectLag(new ArrayList<>()); + try { + supervisor.computeLagStats(); + } + catch (Exception e) { + Assert.assertTrue(e instanceof UnsupportedOperationException); + } - Map supervisorTaskInfos = supervisor.getSupervisorTaskInfos(); - Assert.assertNull(supervisorTaskInfos); + try { + int count = supervisor.getActiveTaskGroupsCount(); + } + catch (Exception e) { + Assert.assertTrue(e instanceof UnsupportedOperationException); + } Callable noop = new Callable() { @Override @@ -221,8 +228,6 @@ public Integer call() return -1; } }; - Runnable runnable = supervisor.buildDynamicAllocationTask(noop); - Assert.assertNull(runnable); } catch (Exception e) { diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java index 26e2acaa5296..a02568a8eeab 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisor.java @@ -38,6 +38,7 @@ import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.TaskMaster; import org.apache.druid.indexing.overlord.TaskStorage; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; import org.apache.druid.indexing.seekablestream.SeekableStreamEndSequenceNumbers; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTask; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskIOConfig; @@ -332,13 +333,14 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() } @Override - public void collectLag(ArrayList lags) + public LagStats computeLagStats() { Map partitionRecordLag = getPartitionRecordLag(); if (partitionRecordLag == null) { - return; + return new LagStats(0, 0, 0); } - computeLags(partitionRecordLag, lags); + + return computeLags(partitionRecordLag); } @Override diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java index 3b2bcc073725..165fad7726f9 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java @@ -51,7 +51,7 @@ public KafkaSupervisorIOConfig( @JsonProperty("taskCount") Integer taskCount, @JsonProperty("taskDuration") Period taskDuration, @JsonProperty("consumerProperties") Map consumerProperties, - @JsonProperty("dynamicAllocationTasksProperties") Map dynamicAllocationTasksProperties, + @JsonProperty("autoscalerConfig") Map autoscalerConfig, @JsonProperty("pollTimeout") Long pollTimeout, @JsonProperty("startDelay") Period startDelay, @JsonProperty("period") Period period, @@ -74,7 +74,7 @@ public KafkaSupervisorIOConfig( completionTimeout, lateMessageRejectionPeriod, earlyMessageRejectionPeriod, - dynamicAllocationTasksProperties, + autoscalerConfig, lateMessageRejectionStartDateTime ); diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 900aca27e05a..f57b8c6bf7be 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -260,21 +260,21 @@ public KafkaIndexTaskClient build( } }; - HashMap dynamicAllocationTasksProperties = new HashMap<>(); - dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", true); - dynamicAllocationTasksProperties.put("metricsCollectionIntervalMillis", 500); - dynamicAllocationTasksProperties.put("metricsCollectionRangeMillis", 500); - dynamicAllocationTasksProperties.put("scaleOutThreshold", 0); - dynamicAllocationTasksProperties.put("triggerScaleOutThresholdFrequency", 0.0); - dynamicAllocationTasksProperties.put("scaleInThreshold", 1000000); - dynamicAllocationTasksProperties.put("triggerScaleInThresholdFrequency", 0.8); - dynamicAllocationTasksProperties.put("dynamicCheckStartDelayMillis", 0); - dynamicAllocationTasksProperties.put("dynamicCheckPeriod", 100); - dynamicAllocationTasksProperties.put("taskCountMax", 2); - dynamicAllocationTasksProperties.put("taskCountMin", 1); - dynamicAllocationTasksProperties.put("scaleInStep", 1); - dynamicAllocationTasksProperties.put("scaleOutStep", 2); - dynamicAllocationTasksProperties.put("minTriggerDynamicFrequencyMillis", 1200000); + HashMap autoscalerConfig = new HashMap<>(); + autoscalerConfig.put("enableTaskAutoscaler", true); + autoscalerConfig.put("metricsCollectionIntervalMillis", 500); + autoscalerConfig.put("metricsCollectionRangeMillis", 500); + autoscalerConfig.put("scaleOutThreshold", 0); + autoscalerConfig.put("triggerScaleOutThresholdFrequency", 0.0); + autoscalerConfig.put("scaleInThreshold", 1000000); + autoscalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoscalerConfig.put("dynamicCheckStartDelayMillis", 0); + autoscalerConfig.put("dynamicCheckPeriod", 100); + autoscalerConfig.put("taskCountMax", 2); + autoscalerConfig.put("taskCountMin", 1); + autoscalerConfig.put("scaleInStep", 1); + autoscalerConfig.put("scaleOutStep", 2); + autoscalerConfig.put("minTriggerDynamicFrequencyMillis", 1200000); final Map consumerProperties = KafkaConsumerConfigs.getConsumerProperties(); consumerProperties.put("myCustomKey", "myCustomValue"); @@ -287,7 +287,7 @@ public KafkaIndexTaskClient build( 1, new Period("PT1H"), consumerProperties, - dynamicAllocationTasksProperties, + autoscalerConfig, KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), @@ -323,6 +323,7 @@ public KafkaIndexTaskClient build( null, null, null, + null, null ); diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java index 14fa97ffe451..6d6bf99fe81e 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java @@ -39,6 +39,7 @@ import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.TaskMaster; import org.apache.druid.indexing.overlord.TaskStorage; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; import org.apache.druid.indexing.seekablestream.SeekableStreamDataSourceMetadata; import org.apache.druid.indexing.seekablestream.SeekableStreamEndSequenceNumbers; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTask; @@ -380,8 +381,9 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() // not yet supported, will be implemented in the future maybe? need to find a proper way to measure kinesis lag. @Override - public void collectLag(ArrayList lags) + public LagStats computeLagStats() { + return null; } @Override diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java index 131698186586..d6c2b3774901 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java @@ -72,7 +72,7 @@ public KinesisSupervisorIOConfig( @JsonProperty("fetchDelayMillis") Integer fetchDelayMillis, @JsonProperty("awsAssumedRoleArn") String awsAssumedRoleArn, @JsonProperty("awsExternalId") String awsExternalId, - @JsonProperty("dynamicAllocationTasksProperties") Map dynamicAllocationTasksProperties, + @JsonProperty("autoscalerConfig") Map autoscalerConfig, @JsonProperty("deaggregate") boolean deaggregate ) { @@ -88,9 +88,16 @@ public KinesisSupervisorIOConfig( completionTimeout, lateMessageRejectionPeriod, earlyMessageRejectionPeriod, - dynamicAllocationTasksProperties, + null, lateMessageRejectionStartDateTime ); + + // for now dynamic Allocation Tasks is not supported here + // throw UnsupportedOperationException in case someone sets this on a kinesis supervisor spec. + if (autoscalerConfig != null || !autoscalerConfig.isEmpty()) { + throw new UnsupportedOperationException("Dynamic Allocation Tasks is not supported here"); + } + this.endpoint = endpoint != null ? endpoint : (region != null ? region.getEndpoint() : KinesisRegion.US_EAST_1.getEndpoint()); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 712627a03c3a..0b5c2389971a 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -59,6 +59,7 @@ import org.apache.druid.indexing.overlord.supervisor.SupervisorManager; import org.apache.druid.indexing.overlord.supervisor.SupervisorReport; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; import org.apache.druid.indexing.seekablestream.SeekableStreamDataSourceMetadata; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTask; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClient; @@ -147,6 +148,9 @@ public abstract class SeekableStreamSupervisor taskClient; private final SeekableStreamSupervisorSpec spec; private final SeekableStreamSupervisorIOConfig ioConfig; - private final Map dynamicAllocationTasksProperties; + private final Map autoscalerConfig; private final SeekableStreamSupervisorTuningConfig tuningConfig; private final SeekableStreamIndexTaskTuningConfig taskTuningConfig; private final String supervisorId; @@ -610,7 +614,7 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private volatile boolean stopped = false; private volatile boolean lifecycleStarted = false; private final ServiceEmitter emitter; - private final boolean enableDynamicAllocationTasks; + private final boolean enableTaskAutoscaler; private int taskCountMax; private long minTriggerDynamicFrequency; @@ -636,16 +640,16 @@ public SeekableStreamSupervisor( this.useExclusiveStartingSequence = useExclusiveStartingSequence; this.dataSource = spec.getDataSchema().getDataSource(); this.ioConfig = spec.getIoConfig(); - this.dynamicAllocationTasksProperties = ioConfig.getDynamicAllocationTasksProperties(); - log.debug("Get dynamicAllocationTasksProperties from IOConfig : [%s] in [%s]", dynamicAllocationTasksProperties, dataSource); - if (dynamicAllocationTasksProperties != null && !dynamicAllocationTasksProperties.isEmpty() && Boolean.parseBoolean(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("enableDynamicAllocationTasks", false)))) { - log.info("EnableDynamicAllocationTasks for datasource [%s]", dataSource); - this.taskCountMax = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 4))); - this.minTriggerDynamicFrequency = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("minTriggerDynamicFrequencyMillis", 600000))); - this.enableDynamicAllocationTasks = true; + this.autoscalerConfig = ioConfig.getautoscalerConfig(); + log.debug("Get autoscalerConfig from IOConfig : [%s] in [%s]", autoscalerConfig, dataSource); + if (autoscalerConfig != null && !autoscalerConfig.isEmpty() && Boolean.parseBoolean(String.valueOf(autoscalerConfig.getOrDefault("enableTaskAutoscaler", true)))) { + log.info("enableTaskAutoscaler for datasource [%s]", dataSource); + this.taskCountMax = Integer.parseInt(String.valueOf(autoscalerConfig.getOrDefault("taskCountMax", TASK_COUNT_MAX))); + this.minTriggerDynamicFrequency = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("minTriggerDynamicFrequencyMillis", 600000))); + this.enableTaskAutoscaler = true; } else { log.info("Disable dynamic allocate tasks for [%s]", dataSource); - this.enableDynamicAllocationTasks = false; + this.enableTaskAutoscaler = false; } this.tuningConfig = spec.getTuningConfig(); @@ -661,7 +665,7 @@ public SeekableStreamSupervisor( ); int workerThreads; - if (enableDynamicAllocationTasks) { + if (enableTaskAutoscaler) { workerThreads = (this.tuningConfig.getWorkerThreads() != null ? this.tuningConfig.getWorkerThreads() : Math.min(10, taskCountMax)); @@ -715,7 +719,7 @@ public Optional getTaskStatus(String id) + IndexTaskClient.MAX_RETRY_WAIT_SECONDS) ); int chatThreads; - if (enableDynamicAllocationTasks) { + if (enableTaskAutoscaler) { chatThreads = (this.tuningConfig.getChatThreads() != null ? this.tuningConfig.getChatThreads() : Math.min(10, taskCountMax * this.ioConfig.getReplicas())); @@ -743,9 +747,9 @@ public Optional getTaskStatus(String id) @Override - public Map getSupervisorTaskInfos() + public int getActiveTaskGroupsCount() { - return activelyReadingTaskGroups; + return activelyReadingTaskGroups.values().size(); } @Override @@ -948,7 +952,6 @@ public void tryInit() } } - @Override public Runnable buildDynamicAllocationTask(Callable scaleAction) { return () -> notices.add(new DynamicAllocationTasksNotice(scaleAction)); @@ -3664,22 +3667,22 @@ protected void emitLag() if (partitionLags == null) { return; } - ArrayList lags = new ArrayList<>(3); - computeLags(partitionLags, lags); + + LagStats lagStats = computeLags(partitionLags); emitter.emit( ServiceMetricEvent.builder() .setDimension("dataSource", dataSource) - .build(StringUtils.format("ingest/%s/lag%s", type, suffix), lags.get(1)) + .build(StringUtils.format("ingest/%s/lag%s", type, suffix), lagStats.getTotalLag()) ); emitter.emit( ServiceMetricEvent.builder() .setDimension("dataSource", dataSource) - .build(StringUtils.format("ingest/%s/maxLag%s", type, suffix), lags.get(0)) + .build(StringUtils.format("ingest/%s/maxLag%s", type, suffix), lagStats.getMaxLag()) ); emitter.emit( ServiceMetricEvent.builder() .setDimension("dataSource", dataSource) - .build(StringUtils.format("ingest/%s/avgLag%s", type, suffix), lags.get(2)) + .build(StringUtils.format("ingest/%s/avgLag%s", type, suffix), lagStats.getAvgLag()) ); }; @@ -3696,9 +3699,8 @@ protected void emitLag() /** * This method compute maxLag, totalLag and avgLag then fill in 'lags' * @param partitionLags lags per partition - * @param lags a arraylist in order of maxLag, totalLag and avgLag */ - protected void computeLags(Map partitionLags, ArrayList lags) + protected LagStats computeLags(Map partitionLags) { long maxLag = 0, totalLag = 0, avgLag; for (long lag : partitionLags.values()) { @@ -3708,9 +3710,7 @@ protected void computeLags(Map partitionLags, ArrayList lateMessageRejectionPeriod; private final Optional earlyMessageRejectionPeriod; private final Optional lateMessageRejectionStartDateTime; - @Nullable private final Map dynamicAllocationTasksProperties; + @Nullable private final Map autoscalerConfig; public SeekableStreamSupervisorIOConfig( String stream, @@ -62,7 +62,7 @@ public SeekableStreamSupervisorIOConfig( Period completionTimeout, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, - @Nullable Map dynamicAllocationTasksProperties, + @Nullable Map autoscalerConfig, DateTime lateMessageRejectionStartDateTime ) { @@ -92,7 +92,7 @@ public SeekableStreamSupervisorIOConfig( + "and lateMessageRejectionPeriod."); } // Could be null - this.dynamicAllocationTasksProperties = dynamicAllocationTasksProperties; + this.autoscalerConfig = autoscalerConfig; } private static Duration defaultDuration(final Period period, final String theDefault) @@ -121,9 +121,9 @@ public Integer getReplicas() @Nullable @JsonProperty - public Map getDynamicAllocationTasksProperties() + public Map getautoscalerConfig() { - return dynamicAllocationTasksProperties; + return autoscalerConfig; } @JsonProperty diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java index d26caddf52b6..d9e2afbca085 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java @@ -167,13 +167,20 @@ public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) { String dataSource = getId(); SupervisorTaskAutoscaler autoScaler = new DummyAutoScaler(supervisor, dataSource); - Map dynamicAllocationTasksProperties = ingestionSchema.getIOConfig().getDynamicAllocationTasksProperties(); - if (dynamicAllocationTasksProperties != null && !dynamicAllocationTasksProperties.isEmpty() && Boolean.parseBoolean(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("enableDynamicAllocationTasks", false)))) { - String autoScalerStrategy = String.valueOf(dynamicAllocationTasksProperties.getOrDefault("autoScalerStrategy", "default")); + Map autoscalerConfig = ingestionSchema.getIOConfig().getautoscalerConfig(); + + // kinesis'autoscalerConfig is always null for now, So that kinesis will hold a DummyAutoScaler. + // only SeekableStreamSupervisor is supported here. + if (autoscalerConfig != null + && !autoscalerConfig.isEmpty() + && Boolean.parseBoolean(String.valueOf(autoscalerConfig.getOrDefault("enableTaskAutoscaler", true))) + && supervisor instanceof SeekableStreamSupervisor) { + + String autoScalerStrategy = String.valueOf(autoscalerConfig.getOrDefault("autoScalerStrategy", "default")); // will thorw 'Return value of String.hashCode() ignored : RV_RETURN_VALUE_IGNORED' just Suppress it. switch (StringUtils.toLowerCase(autoScalerStrategy)) { - default: autoScaler = new DefaultAutoScaler(supervisor, dataSource, dynamicAllocationTasksProperties, this); + default: autoScaler = new DefaultAutoScaler(supervisor, dataSource, autoscalerConfig, this); } } return autoScaler; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java index abcb9360a586..77cd796d8100 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java @@ -22,7 +22,9 @@ import org.apache.commons.collections4.queue.CircularFifoQueue; import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisor; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.concurrent.Execs; import org.apache.druid.java.util.emitter.EmittingLogger; @@ -41,7 +43,6 @@ public class DefaultAutoScaler implements SupervisorTaskAutoscaler private final String dataSource; private final long metricsCollectionIntervalMillis; private final long metricsCollectionRangeMillis; - private final Map supervisorTaskInfos; private final CircularFifoQueue lagMetricsQueue; private final long dynamicCheckStartDelayMillis; private final long dynamicCheckPeriod; @@ -56,35 +57,34 @@ public class DefaultAutoScaler implements SupervisorTaskAutoscaler private final ScheduledExecutorService lagComputationExec; private final ScheduledExecutorService allocationExec; private final SupervisorSpec spec; - private final Supervisor supervisor; + private final SeekableStreamSupervisor supervisor; private static ReentrantLock lock = new ReentrantLock(true); - public DefaultAutoScaler(Supervisor supervisor, String dataSource, Map dynamicAllocationTasksProperties, SupervisorSpec spec) + public DefaultAutoScaler(Supervisor supervisor, String dataSource, Map autoscalerConfig, SupervisorSpec spec) { String supervisorId = StringUtils.format("KafkaSupervisor-%s", dataSource); this.dataSource = dataSource; - this.metricsCollectionIntervalMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionIntervalMillis", 30000))); - this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("metricsCollectionRangeMillis", 600000))); + this.metricsCollectionIntervalMillis = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("metricsCollectionIntervalMillis", 30000))); + this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("metricsCollectionRangeMillis", 600000))); int slots = (int) (metricsCollectionRangeMillis / metricsCollectionIntervalMillis) + 1; log.debug(" The interval of metrics collection is [%s], [%s] timeRange will collect [%s] data points for dataSource [%s].", metricsCollectionIntervalMillis, metricsCollectionRangeMillis, slots, dataSource); this.lagMetricsQueue = new CircularFifoQueue<>(slots); - this.dynamicCheckStartDelayMillis = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckStartDelayMillis", 300000))); - this.dynamicCheckPeriod = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("dynamicCheckPeriod", 60000))); - this.scaleOutThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutThreshold", 6000000))); - this.scaleInThreshold = Long.parseLong(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInThreshold", 1000000))); - this.triggerScaleOutThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerScaleOutThresholdFrequency", 0.3))); - this.triggerScaleInThresholdFrequency = Double.parseDouble(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("triggerScaleInThresholdFrequency", 0.9))); - this.taskCountMax = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMax", 4))); - this.taskCountMin = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("taskCountMin", 1))); - this.scaleInStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleInStep", 1))); - this.scaleOutStep = Integer.parseInt(String.valueOf(dynamicAllocationTasksProperties.getOrDefault("scaleOutStep", 2))); + this.dynamicCheckStartDelayMillis = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("dynamicCheckStartDelayMillis", 300000))); + this.dynamicCheckPeriod = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("dynamicCheckPeriod", 60000))); + this.scaleOutThreshold = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("scaleOutThreshold", 6000000))); + this.scaleInThreshold = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("scaleInThreshold", 1000000))); + this.triggerScaleOutThresholdFrequency = Double.parseDouble(String.valueOf(autoscalerConfig.getOrDefault("triggerScaleOutThresholdFrequency", 0.3))); + this.triggerScaleInThresholdFrequency = Double.parseDouble(String.valueOf(autoscalerConfig.getOrDefault("triggerScaleInThresholdFrequency", 0.9))); + this.taskCountMax = Integer.parseInt(String.valueOf(autoscalerConfig.getOrDefault("taskCountMax", SeekableStreamSupervisor.TASK_COUNT_MAX))); + this.taskCountMin = Integer.parseInt(String.valueOf(autoscalerConfig.getOrDefault("taskCountMin", SeekableStreamSupervisor.TASK_COUNT_MIN))); + this.scaleInStep = Integer.parseInt(String.valueOf(autoscalerConfig.getOrDefault("scaleInStep", 1))); + this.scaleOutStep = Integer.parseInt(String.valueOf(autoscalerConfig.getOrDefault("scaleOutStep", 2))); this.allocationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Allocation-%d"); this.lagComputationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Computation-%d"); this.spec = spec; - this.supervisor = supervisor; - this.supervisorTaskInfos = supervisor.getSupervisorTaskInfos(); + this.supervisor = (SeekableStreamSupervisor) supervisor; } @Override @@ -113,7 +113,7 @@ public Integer call() } }; - log.info("EnableDynamicAllocationTasks for datasource [%s]", dataSource); + log.info("enableTaskAutoscaler for datasource [%s]", dataSource); log.debug("Collect and compute lags at fixed rate of [%s] for dataSource[%s].", metricsCollectionIntervalMillis, dataSource); lagComputationExec.scheduleAtFixedRate( collectAndComputeLags(), @@ -168,17 +168,21 @@ public void run() lock.lock(); try { if (!spec.isSuspended()) { - ArrayList metricsInfo = new ArrayList<>(3); - supervisor.collectLag(metricsInfo); - long totalLags = metricsInfo.size() < 3 ? 0 : metricsInfo.get(1); - lagMetricsQueue.offer(totalLags > 0 ? totalLags : 0); + LagStats lagStats = supervisor.computeLagStats(); + if (lagStats == null) { + lagMetricsQueue.offer(0L); + } else { + long totalLags = lagStats.getTotalLag(); + lagMetricsQueue.offer(totalLags > 0 ? totalLags : 0L); + } + log.debug("Current lag metric points [%s] for dataSource [%s].", new ArrayList<>(lagMetricsQueue), dataSource); } else { log.debug("[%s] supervisor is suspended, skip to collect kafka lags", dataSource); } } catch (Exception e) { - log.warn(e, "Error, When collect kafka lags"); + log.warn(e, "Error, When collect lags"); } finally { lock.unlock(); @@ -220,9 +224,12 @@ private Integer computeDesireTaskCount(List lags) log.debug("triggerScaleOutThresholdFrequency is [%s] and triggerScaleInThresholdFrequency is [%s] for dataSource [%s].", triggerScaleOutThresholdFrequency, triggerScaleInThresholdFrequency, dataSource); log.info("beyondProportion is [%s] and withinProportion is [%s] for dataSource [%s].", beyondProportion, withinProportion, dataSource); - int currentActiveTaskCount; + int currentActiveTaskCount = supervisor.getActiveTaskGroupsCount(); + if (currentActiveTaskCount <= 0) { + log.info("CurrentActiveTaskCount is lower than 0 ??? skip [%s].", dataSource); + return -1; + } int desireActiveTaskCount; - currentActiveTaskCount = supervisorTaskInfos.values().size(); if (beyondProportion >= triggerScaleOutThresholdFrequency) { // Do Scale out diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 69a50c282adf..f0a077ef3cb1 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -36,6 +36,7 @@ import org.apache.druid.indexing.overlord.TaskStorage; import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.indexing.seekablestream.common.OrderedSequenceNumber; import org.apache.druid.indexing.seekablestream.common.RecordSupplier; @@ -333,8 +334,9 @@ protected void scheduleReporting(ScheduledExecutorService reportingExec) } @Override - public void collectLag(ArrayList lags) + public LagStats computeLagStats() { + return null; } } @@ -483,6 +485,7 @@ public SeekableStreamIndexTaskTuningConfig convertToTaskTuningConfig() null, null, null, + null, null ) { @@ -505,31 +508,31 @@ public String toString() @Test public void testAutoScalerCreated() { - HashMap dynamicAllocationTasksProperties = new HashMap<>(); - dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", true); - dynamicAllocationTasksProperties.put("metricsCollectionIntervalMillis", 500); - dynamicAllocationTasksProperties.put("metricsCollectionRangeMillis", 500); - dynamicAllocationTasksProperties.put("scaleOutThreshold", 5000000); - dynamicAllocationTasksProperties.put("triggerScaleOutThresholdFrequency", 0.3); - dynamicAllocationTasksProperties.put("scaleInThreshold", 1000000); - dynamicAllocationTasksProperties.put("triggerScaleInThresholdFrequency", 0.8); - dynamicAllocationTasksProperties.put("dynamicCheckStartDelayMillis", 0); - dynamicAllocationTasksProperties.put("dynamicCheckPeriod", 100); - dynamicAllocationTasksProperties.put("taskCountMax", 8); - dynamicAllocationTasksProperties.put("taskCountMin", 1); - dynamicAllocationTasksProperties.put("scaleInStep", 1); - dynamicAllocationTasksProperties.put("scaleOutStep", 2); - dynamicAllocationTasksProperties.put("minTriggerDynamicFrequencyMillis", 1200000); + HashMap autoscalerConfig = new HashMap<>(); + autoscalerConfig.put("enableTaskAutoscaler", true); + autoscalerConfig.put("metricsCollectionIntervalMillis", 500); + autoscalerConfig.put("metricsCollectionRangeMillis", 500); + autoscalerConfig.put("scaleOutThreshold", 5000000); + autoscalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); + autoscalerConfig.put("scaleInThreshold", 1000000); + autoscalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoscalerConfig.put("dynamicCheckStartDelayMillis", 0); + autoscalerConfig.put("dynamicCheckPeriod", 100); + autoscalerConfig.put("taskCountMax", 8); + autoscalerConfig.put("taskCountMin", 1); + autoscalerConfig.put("scaleInStep", 1); + autoscalerConfig.put("scaleOutStep", 2); + autoscalerConfig.put("minTriggerDynamicFrequencyMillis", 1200000); EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes(); EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes(); EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getDynamicAllocationTasksProperties()).andReturn(dynamicAllocationTasksProperties).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getautoscalerConfig()).andReturn(autoscalerConfig).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); - EasyMock.expect(supervisor4.getSupervisorTaskInfos()).andReturn(new HashMap()).anyTimes(); + EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); EasyMock.replay(supervisor4); TesstSeekableStreamSupervisorSpec spec = new TesstSeekableStreamSupervisorSpec(ingestionSchema, @@ -549,11 +552,11 @@ public void testAutoScalerCreated() SupervisorTaskAutoscaler autoscaler = spec.createAutoscaler(supervisor4); Assert.assertTrue(autoscaler instanceof DefaultAutoScaler); - dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", false); + autoscalerConfig.put("enableTaskAutoscaler", false); SupervisorTaskAutoscaler autoscaler2 = spec.createAutoscaler(supervisor4); Assert.assertTrue(autoscaler2 instanceof DummyAutoScaler); - dynamicAllocationTasksProperties.remove("enableDynamicAllocationTasks"); + autoscalerConfig.remove("enableTaskAutoscaler"); SupervisorTaskAutoscaler autoscaler3 = spec.createAutoscaler(supervisor4); Assert.assertTrue(autoscaler3 instanceof DummyAutoScaler); @@ -743,42 +746,42 @@ null, getScaleInProperties(), null private static Map getScaleOutProperties() { - HashMap dynamicAllocationTasksProperties = new HashMap<>(); - dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", true); - dynamicAllocationTasksProperties.put("metricsCollectionIntervalMillis", 500); - dynamicAllocationTasksProperties.put("metricsCollectionRangeMillis", 500); - dynamicAllocationTasksProperties.put("scaleOutThreshold", 0); - dynamicAllocationTasksProperties.put("triggerScaleOutThresholdFrequency", 0.0); - dynamicAllocationTasksProperties.put("scaleInThreshold", 1000000); - dynamicAllocationTasksProperties.put("triggerScaleInThresholdFrequency", 0.8); - dynamicAllocationTasksProperties.put("dynamicCheckStartDelayMillis", 0); - dynamicAllocationTasksProperties.put("dynamicCheckPeriod", 100); - dynamicAllocationTasksProperties.put("taskCountMax", 2); - dynamicAllocationTasksProperties.put("taskCountMin", 1); - dynamicAllocationTasksProperties.put("scaleInStep", 1); - dynamicAllocationTasksProperties.put("scaleOutStep", 2); - dynamicAllocationTasksProperties.put("minTriggerDynamicFrequencyMillis", 1200000); - return dynamicAllocationTasksProperties; + HashMap autoscalerConfig = new HashMap<>(); + autoscalerConfig.put("enableTaskAutoscaler", true); + autoscalerConfig.put("metricsCollectionIntervalMillis", 500); + autoscalerConfig.put("metricsCollectionRangeMillis", 500); + autoscalerConfig.put("scaleOutThreshold", 0); + autoscalerConfig.put("triggerScaleOutThresholdFrequency", 0.0); + autoscalerConfig.put("scaleInThreshold", 1000000); + autoscalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoscalerConfig.put("dynamicCheckStartDelayMillis", 0); + autoscalerConfig.put("dynamicCheckPeriod", 100); + autoscalerConfig.put("taskCountMax", 2); + autoscalerConfig.put("taskCountMin", 1); + autoscalerConfig.put("scaleInStep", 1); + autoscalerConfig.put("scaleOutStep", 2); + autoscalerConfig.put("minTriggerDynamicFrequencyMillis", 1200000); + return autoscalerConfig; } private static Map getScaleInProperties() { - HashMap dynamicAllocationTasksProperties = new HashMap<>(); - dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", true); - dynamicAllocationTasksProperties.put("metricsCollectionIntervalMillis", 500); - dynamicAllocationTasksProperties.put("metricsCollectionRangeMillis", 500); - dynamicAllocationTasksProperties.put("scaleOutThreshold", 8000000); - dynamicAllocationTasksProperties.put("triggerScaleOutThresholdFrequency", 0.3); - dynamicAllocationTasksProperties.put("scaleInThreshold", 0); - dynamicAllocationTasksProperties.put("triggerScaleInThresholdFrequency", 0.0); - dynamicAllocationTasksProperties.put("dynamicCheckStartDelayMillis", 0); - dynamicAllocationTasksProperties.put("dynamicCheckPeriod", 100); - dynamicAllocationTasksProperties.put("taskCountMax", 2); - dynamicAllocationTasksProperties.put("taskCountMin", 1); - dynamicAllocationTasksProperties.put("scaleInStep", 1); - dynamicAllocationTasksProperties.put("scaleOutStep", 2); - dynamicAllocationTasksProperties.put("minTriggerDynamicFrequencyMillis", 1200000); - return dynamicAllocationTasksProperties; + HashMap autoscalerConfig = new HashMap<>(); + autoscalerConfig.put("enableTaskAutoscaler", true); + autoscalerConfig.put("metricsCollectionIntervalMillis", 500); + autoscalerConfig.put("metricsCollectionRangeMillis", 500); + autoscalerConfig.put("scaleOutThreshold", 8000000); + autoscalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); + autoscalerConfig.put("scaleInThreshold", 0); + autoscalerConfig.put("triggerScaleInThresholdFrequency", 0.0); + autoscalerConfig.put("dynamicCheckStartDelayMillis", 0); + autoscalerConfig.put("dynamicCheckPeriod", 100); + autoscalerConfig.put("taskCountMax", 2); + autoscalerConfig.put("taskCountMin", 1); + autoscalerConfig.put("scaleInStep", 1); + autoscalerConfig.put("scaleOutStep", 2); + autoscalerConfig.put("minTriggerDynamicFrequencyMillis", 1200000); + return autoscalerConfig; } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index 9ecf12a85cc9..2e15f0485063 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -43,6 +43,7 @@ import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager.BasicState; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; import org.apache.druid.indexing.seekablestream.SeekableStreamDataSourceMetadata; import org.apache.druid.indexing.seekablestream.SeekableStreamEndSequenceNumbers; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTask; @@ -836,22 +837,22 @@ null, getProperties(), null private static Map getProperties() { - HashMap dynamicAllocationTasksProperties = new HashMap<>(); - dynamicAllocationTasksProperties.put("enableDynamicAllocationTasks", true); - dynamicAllocationTasksProperties.put("metricsCollectionIntervalMillis", 500); - dynamicAllocationTasksProperties.put("metricsCollectionRangeMillis", 500); - dynamicAllocationTasksProperties.put("scaleOutThreshold", 5000000); - dynamicAllocationTasksProperties.put("triggerScaleOutThresholdFrequency", 0.3); - dynamicAllocationTasksProperties.put("scaleInThreshold", 1000000); - dynamicAllocationTasksProperties.put("triggerScaleInThresholdFrequency", 0.8); - dynamicAllocationTasksProperties.put("dynamicCheckStartDelayMillis", 0); - dynamicAllocationTasksProperties.put("dynamicCheckPeriod", 100); - dynamicAllocationTasksProperties.put("taskCountMax", 8); - dynamicAllocationTasksProperties.put("taskCountMin", 1); - dynamicAllocationTasksProperties.put("scaleInStep", 1); - dynamicAllocationTasksProperties.put("scaleOutStep", 2); - dynamicAllocationTasksProperties.put("minTriggerDynamicFrequencyMillis", 1200000); - return dynamicAllocationTasksProperties; + HashMap autoscalerConfig = new HashMap<>(); + autoscalerConfig.put("enableTaskAutoscaler", true); + autoscalerConfig.put("metricsCollectionIntervalMillis", 500); + autoscalerConfig.put("metricsCollectionRangeMillis", 500); + autoscalerConfig.put("scaleOutThreshold", 5000000); + autoscalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); + autoscalerConfig.put("scaleInThreshold", 1000000); + autoscalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoscalerConfig.put("dynamicCheckStartDelayMillis", 0); + autoscalerConfig.put("dynamicCheckPeriod", 100); + autoscalerConfig.put("taskCountMax", 8); + autoscalerConfig.put("taskCountMin", 1); + autoscalerConfig.put("scaleInStep", 1); + autoscalerConfig.put("scaleOutStep", 2); + autoscalerConfig.put("minTriggerDynamicFrequencyMillis", 1200000); + return autoscalerConfig; } private static SeekableStreamSupervisorTuningConfig getTuningConfig() @@ -1202,8 +1203,9 @@ protected void scheduleReporting(ScheduledExecutorService reportingExec) } @Override - public void collectLag(ArrayList lags) + public LagStats computeLagStats() { + return null; } } @@ -1248,8 +1250,9 @@ protected void emitLag() } @Override - public void collectLag(ArrayList lags) + public LagStats computeLagStats() { + return null; } @Override diff --git a/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json b/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json index 7f6c50817b62..49428cceec48 100644 --- a/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json +++ b/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json @@ -48,8 +48,8 @@ "ioConfig": { "%%TOPIC_KEY%%": "%%TOPIC_VALUE%%", "%%STREAM_PROPERTIES_KEY%%": %%STREAM_PROPERTIES_VALUE%%, - "dynamicAllocationTasksProperties": { - "enableDynamicAllocationTasks": true, + "autoscalerConfig": { + "enableTaskAutoscaler": true, "metricsCollectionIntervalMillis": 500, "metricsCollectionRangeMillis": 500, "scaleOutThreshold": 0, diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java index e128e3766e8e..0bd5399cb699 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java @@ -23,12 +23,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.annotations.VisibleForTesting; import org.apache.druid.indexing.overlord.DataSourceMetadata; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.concurrent.Callable; @@ -159,30 +158,19 @@ public void checkpoint(int taskGroupId, DataSourceMetadata checkpointMetadata) } @Override - public void collectLag(ArrayList lags) - { - } - - @Override - public Runnable buildDynamicAllocationTask(Callable scaleAction) + public LagStats computeLagStats() { return null; } @Override - public Map getSupervisorTaskInfos() + public int getActiveTaskGroupsCount() { - return null; + return -1; } }; } - @Override - public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) - { - return null; - } - @Override public SupervisorSpec createRunningSpec() { diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java index b4df1c2c12f2..b345163b2c32 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java @@ -21,11 +21,10 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.indexing.overlord.DataSourceMetadata; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; import javax.annotation.Nullable; -import java.util.ArrayList; import java.util.Map; -import java.util.concurrent.Callable; public interface Supervisor { @@ -68,16 +67,10 @@ default Boolean isHealthy() void checkpoint(int taskGroupId, DataSourceMetadata checkpointMetadata); /** - * Collect maxLag, totalLag, avgLag into ArrayList lags + * Collect maxLag, totalLag, avgLag * Only support Kafka ingestion so far. - * @param lags , Notice : The order of values is maxLag, totalLag and avgLag. */ - void collectLag(ArrayList lags); + LagStats computeLagStats(); - /** - * use for autoscaler - */ - Runnable buildDynamicAllocationTask(Callable scaleAction); - - Map getSupervisorTaskInfos(); + int getActiveTaskGroupsCount(); } diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java index adc462f42ce9..5b5137a4f76f 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java @@ -41,7 +41,10 @@ public interface SupervisorSpec */ Supervisor createSupervisor(); - SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor); + default SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + { + return null; + } List getDataSources(); diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/LagStats.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/LagStats.java new file mode 100644 index 000000000000..7b6e5fd0bab1 --- /dev/null +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/LagStats.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.overlord.supervisor.autoscaler; + +public class LagStats +{ + private final long maxLag; + private final long totalLag; + private final long avgLag; + + public LagStats(long maxLag, long totalLag, long avgLag) + { + this.maxLag = maxLag; + this.totalLag = totalLag; + this.avgLag = avgLag; + } + + public long getMaxLag() + { + return maxLag; + } + + public long getTotalLag() + { + return totalLag; + } + + public long getAvgLag() + { + return avgLag; + } +} diff --git a/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java b/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java index 9d5eb6b27fbe..af20c38da7ef 100644 --- a/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java +++ b/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java @@ -25,9 +25,7 @@ import org.junit.Assert; import org.junit.Test; -import java.util.ArrayList; import java.util.Collections; -import java.util.Map; import java.util.concurrent.Callable; public class NoopSupervisorSpecTest @@ -48,13 +46,11 @@ public Integer call() return -1; } }; - Runnable runnable = supervisor.buildDynamicAllocationTask(noop); - Assert.assertNull(runnable); - Map supervisorTaskInfos = supervisor.getSupervisorTaskInfos(); - Assert.assertNull(supervisorTaskInfos); + int count = supervisor.getActiveTaskGroupsCount(); + Assert.assertEquals(count, -1); - supervisor.collectLag(new ArrayList()); + supervisor.computeLagStats(); } catch (Exception ex) { e = ex; From 71bdfbbad7c8ff87de78d388dd65c2a25903ca3e Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 2 Feb 2021 15:49:58 +0800 Subject: [PATCH 25/47] code review --- .idea/misc.xml | 4 ++-- docs/development/extensions-core/kafka-ingestion.md | 4 ++-- .../supervisor/SeekableStreamSupervisor.java | 2 +- .../SeekableStreamSupervisorIOConfig.java | 2 +- .../supervisor/SeekableStreamSupervisorSpec.java | 2 +- .../SeekableStreamSupervisorSpecTest.java | 13 +++++++++++-- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 28cdd807c98a..af0408b354b4 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -146,9 +146,9 @@ A sample supervisor spec is shown below: |`lateMessageRejectionStartDateTime`|ISO8601 DateTime|Configure tasks to reject messages with timestamps earlier than this date time; for example if this is set to `2016-01-01T11:00Z` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline).|no (default == none)| |`lateMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps earlier than this period before the task was created; for example if this is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline). Please note that only one of `lateMessageRejectionPeriod` or `lateMessageRejectionStartDateTime` can be specified.|no (default == none)| |`earlyMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps later than this period after the task reached its taskDuration; for example if this is set to `PT1H`, the taskDuration is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps later than *2016-01-01T14:00Z* will be dropped. **Note:** Tasks sometimes run past their task duration, for example, in cases of supervisor failover. Setting earlyMessageRejectionPeriod too low may cause messages to be dropped unexpectedly whenever a task runs past its originally configured task duration.|no (default == none)| -|`autoscalerConfig`|Object|`autoscalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for kafka indexing as of now. See [Dynamic Allocation Tasks Properties](#Dynamic Allocation Tasks Properties) for details.|no (default == null)| +|`autoscalerConfig`|Object|`autoscalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for kafka indexing as of now. See [Tasks Autoscaler Properties](#Tasks Autoscaler Properties) for details.|no (default == null)| -#### Dynamic Allocation Tasks Properties +#### Tasks Autoscaler Properties | Property | Description | Default | | ------------- | ------------- | ------------- | diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 0b5c2389971a..0ec42b6980f4 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -640,7 +640,7 @@ public SeekableStreamSupervisor( this.useExclusiveStartingSequence = useExclusiveStartingSequence; this.dataSource = spec.getDataSchema().getDataSource(); this.ioConfig = spec.getIoConfig(); - this.autoscalerConfig = ioConfig.getautoscalerConfig(); + this.autoscalerConfig = ioConfig.getAutoscalerConfig(); log.debug("Get autoscalerConfig from IOConfig : [%s] in [%s]", autoscalerConfig, dataSource); if (autoscalerConfig != null && !autoscalerConfig.isEmpty() && Boolean.parseBoolean(String.valueOf(autoscalerConfig.getOrDefault("enableTaskAutoscaler", true)))) { log.info("enableTaskAutoscaler for datasource [%s]", dataSource); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java index 754b1e7f08a5..92af2be81852 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java @@ -121,7 +121,7 @@ public Integer getReplicas() @Nullable @JsonProperty - public Map getautoscalerConfig() + public Map getAutoscalerConfig() { return autoscalerConfig; } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java index d9e2afbca085..d4b63eabf563 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java @@ -167,7 +167,7 @@ public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) { String dataSource = getId(); SupervisorTaskAutoscaler autoScaler = new DummyAutoScaler(supervisor, dataSource); - Map autoscalerConfig = ingestionSchema.getIOConfig().getautoscalerConfig(); + Map autoscalerConfig = ingestionSchema.getIOConfig().getAutoscalerConfig(); // kinesis'autoscalerConfig is always null for now, So that kinesis will hold a DummyAutoScaler. // only SeekableStreamSupervisor is supported here. diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index f0a077ef3cb1..58167875b522 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -336,7 +336,7 @@ protected void scheduleReporting(ScheduledExecutorService reportingExec) @Override public LagStats computeLagStats() { - return null; + return new LagStats(0, 0, 0); } } @@ -529,7 +529,7 @@ public void testAutoScalerCreated() EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getautoscalerConfig()).andReturn(autoscalerConfig).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(autoscalerConfig).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); @@ -585,6 +585,15 @@ public void testSeekableStreamSupervisorSpecWithScaleOut() throws InterruptedExc EasyMock.replay(taskMaster); TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); + + LagStats lagStats = supervisor.computeLagStats(); + long totalLag = lagStats.getTotalLag(); + long avgLag = lagStats.getAvgLag(); + long maxLag = lagStats.getMaxLag(); + Assert.assertEquals(totalLag, 0); + Assert.assertEquals(avgLag, 0); + Assert.assertEquals(maxLag, 0); + DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, getScaleOutProperties(), spec); supervisor.start(); autoScaler.start(); From 96025755975a2de106d7a9aa1930d48b17bc25b6 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 2 Feb 2021 17:10:02 +0800 Subject: [PATCH 26/47] remove unused imports --- .idea/misc.xml | 4 ++-- .../druid/indexing/kafka/supervisor/KafkaSupervisorTest.java | 2 +- .../indexing/overlord/supervisor/NoopSupervisorSpec.java | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index f57b8c6bf7be..65bb84a022e7 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -302,6 +302,7 @@ public KafkaIndexTaskClient build( null, 1000, null, + null, 50000, null, new Period("P1Y"), @@ -323,7 +324,6 @@ public KafkaIndexTaskClient build( null, null, null, - null, null ); diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java index 0bd5399cb699..f4227b0256f8 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.concurrent.Callable; /** * Used as a tombstone marker in the supervisors metadata table to indicate that the supervisor has been removed. From 6bbbf297d1ab84b675d07108f554bd765206ff08 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 2 Feb 2021 18:23:52 +0800 Subject: [PATCH 27/47] fix NLP --- .idea/misc.xml | 4 +- .../supervisor/KinesisSupervisorIOConfig.java | 4 +- .../supervisor/KinesisSupervisorTest.java | 65 +++++++++++++++++++ .../autoscaler/DefaultAutoScaler.java | 2 +- .../SeekableStreamSupervisorSpecTest.java | 2 +- 5 files changed, 71 insertions(+), 6 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java index d6c2b3774901..1460dc3d44e3 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java @@ -94,8 +94,8 @@ public KinesisSupervisorIOConfig( // for now dynamic Allocation Tasks is not supported here // throw UnsupportedOperationException in case someone sets this on a kinesis supervisor spec. - if (autoscalerConfig != null || !autoscalerConfig.isEmpty()) { - throw new UnsupportedOperationException("Dynamic Allocation Tasks is not supported here"); + if (autoscalerConfig != null) { + throw new UnsupportedOperationException("Tasks auto scaler for kinesis is not supported yet."); } this.endpoint = endpoint != null diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java index 89093c09e440..2aaca0c4cf73 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java @@ -102,6 +102,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -345,6 +346,70 @@ public void testRecordSupplier() Assert.assertFalse(supplier.isBackgroundFetchRunning()); } + @Test + public void testKinesisIOConfig() + { + Exception e = null; + try { + KinesisSupervisorIOConfig kinesisSupervisorIOConfig = new KinesisSupervisorIOConfig( + STREAM, + INPUT_FORMAT, + "awsEndpoint", + null, + 1, + 1, + new Period("PT30M"), + new Period("P1D"), + new Period("PT30S"), + false, + new Period("PT30M"), + null, + null, + null, + 100, + 1000, + null, + null, + null, + false + ); + } + catch (Exception ex) { + e = ex; + } + Assert.assertNull(e); + + try { + KinesisSupervisorIOConfig kinesisSupervisorIOConfig = new KinesisSupervisorIOConfig( + STREAM, + INPUT_FORMAT, + "awsEndpoint", + null, + 1, + 1, + new Period("PT30M"), + new Period("P1D"), + new Period("PT30S"), + false, + new Period("PT30M"), + null, + null, + null, + 100, + 1000, + null, + null, + new HashMap<>(), + false + ); + } + catch (Exception ex) { + e = ex; + } + Assert.assertNotNull(e); + Assert.assertTrue(e instanceof UnsupportedOperationException); + } + @Test public void testMultiTask() throws Exception { diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java index 77cd796d8100..4939b8de09ba 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java @@ -225,7 +225,7 @@ private Integer computeDesireTaskCount(List lags) log.info("beyondProportion is [%s] and withinProportion is [%s] for dataSource [%s].", beyondProportion, withinProportion, dataSource); int currentActiveTaskCount = supervisor.getActiveTaskGroupsCount(); - if (currentActiveTaskCount <= 0) { + if (currentActiveTaskCount < 0) { log.info("CurrentActiveTaskCount is lower than 0 ??? skip [%s].", dataSource); return -1; } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 58167875b522..960e8adf070a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -558,7 +558,7 @@ public void testAutoScalerCreated() autoscalerConfig.remove("enableTaskAutoscaler"); SupervisorTaskAutoscaler autoscaler3 = spec.createAutoscaler(supervisor4); - Assert.assertTrue(autoscaler3 instanceof DummyAutoScaler); + Assert.assertTrue(autoscaler3 instanceof DefaultAutoScaler); } From 0ae6a34821939049e6cae6312c6f8b796eb4af11 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 3 Feb 2021 10:20:38 +0800 Subject: [PATCH 28/47] fix docs and UTs --- .idea/misc.xml | 4 ++-- docs/development/extensions-core/kafka-ingestion.md | 4 ++-- .../seekablestream/SeekableStreamSupervisorSpecTest.java | 6 ------ .../indexing/overlord/supervisor/NoopSupervisorSpec.java | 2 +- .../apache/druid/indexing/NoopSupervisorSpecTest.java | 9 ++++++++- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index af0408b354b4..5af4084ab460 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -146,13 +146,13 @@ A sample supervisor spec is shown below: |`lateMessageRejectionStartDateTime`|ISO8601 DateTime|Configure tasks to reject messages with timestamps earlier than this date time; for example if this is set to `2016-01-01T11:00Z` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline).|no (default == none)| |`lateMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps earlier than this period before the task was created; for example if this is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline). Please note that only one of `lateMessageRejectionPeriod` or `lateMessageRejectionStartDateTime` can be specified.|no (default == none)| |`earlyMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps later than this period after the task reached its taskDuration; for example if this is set to `PT1H`, the taskDuration is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps later than *2016-01-01T14:00Z* will be dropped. **Note:** Tasks sometimes run past their task duration, for example, in cases of supervisor failover. Setting earlyMessageRejectionPeriod too low may cause messages to be dropped unexpectedly whenever a task runs past its originally configured task duration.|no (default == none)| -|`autoscalerConfig`|Object|`autoscalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for kafka indexing as of now. See [Tasks Autoscaler Properties](#Tasks Autoscaler Properties) for details.|no (default == null)| +|`autoscalerConfig`|Object|`autoscalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for Kafka indexing as of now. See [Tasks Autoscaler Properties](#Tasks Autoscaler Properties) for details.|no (default == null)| #### Tasks Autoscaler Properties | Property | Description | Default | | ------------- | ------------- | ------------- | -| `enableTaskAutoscaler` | whether enable this feature or not. Set false here will disable autoscaler even though autoscalerConfig is not null| true | +| `enableTaskAutoscaler` | whether enable this feature or not. Set false here will disable `autoscaler` even though autoscalerConfig is not null| true | | `metricsCollectionIntervalMillis` | Define the frequency of lag points collection. | 30000 | | `metricsCollectionRangeMillis` | The total time window of lag collection, Use with `metricsCollectionIntervalMillis`，it means that in the recent `metricsCollectionRangeMill`, collect lag metric points every `metricsCollectionIntervalMillis`. | 600000 | | `scaleOutThreshold` | The Threshold of scale out action | 6000000 | diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 960e8adf070a..587c83f5cf30 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -587,12 +587,6 @@ public void testSeekableStreamSupervisorSpecWithScaleOut() throws InterruptedExc TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); LagStats lagStats = supervisor.computeLagStats(); - long totalLag = lagStats.getTotalLag(); - long avgLag = lagStats.getAvgLag(); - long maxLag = lagStats.getMaxLag(); - Assert.assertEquals(totalLag, 0); - Assert.assertEquals(avgLag, 0); - Assert.assertEquals(maxLag, 0); DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, getScaleOutProperties(), spec); supervisor.start(); diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java index f4227b0256f8..b19aeaa28812 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/NoopSupervisorSpec.java @@ -159,7 +159,7 @@ public void checkpoint(int taskGroupId, DataSourceMetadata checkpointMetadata) @Override public LagStats computeLagStats() { - return null; + return new LagStats(0, 0, 0); } @Override diff --git a/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java b/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java index af20c38da7ef..ad7f9699bd72 100644 --- a/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java +++ b/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java @@ -21,6 +21,7 @@ import org.apache.druid.indexing.overlord.supervisor.NoopSupervisorSpec; import org.apache.druid.indexing.overlord.supervisor.Supervisor; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.junit.Assert; import org.junit.Test; @@ -50,7 +51,13 @@ public Integer call() int count = supervisor.getActiveTaskGroupsCount(); Assert.assertEquals(count, -1); - supervisor.computeLagStats(); + LagStats lagStats = supervisor.computeLagStats(); + long totalLag = lagStats.getTotalLag(); + long avgLag = lagStats.getAvgLag(); + long maxLag = lagStats.getMaxLag(); + Assert.assertEquals(totalLag, 0); + Assert.assertEquals(avgLag, 0); + Assert.assertEquals(maxLag, 0); } catch (Exception ex) { e = ex; From 16e4f47421c442851a84c50326d39b3283234094 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 3 Feb 2021 10:24:43 +0800 Subject: [PATCH 29/47] revert misc.xml --- .idea/misc.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + From 25fec0ff18d0acb79734dd93bedccff3ab43308a Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 3 Feb 2021 19:06:24 +0800 Subject: [PATCH 30/47] use jackson to build autoScaleConfig with default values --- .idea/misc.xml | 4 +- .../supervisor/SeekableStreamSupervisor.java | 55 +++---- .../SeekableStreamSupervisorSpec.java | 19 ++- .../autoscaler/AutoScalerConfig.java | 149 ++++++++++++++++++ .../autoscaler/DefaultAutoScaler.java | 72 ++++----- .../SeekableStreamSupervisorSpecTest.java | 58 ++++++- 6 files changed, 268 insertions(+), 89 deletions(-) create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 0ec42b6980f4..eb2797cc5865 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -73,6 +73,7 @@ import org.apache.druid.indexing.seekablestream.common.RecordSupplier; import org.apache.druid.indexing.seekablestream.common.StreamException; import org.apache.druid.indexing.seekablestream.common.StreamPartition; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; @@ -148,9 +149,6 @@ public abstract class SeekableStreamSupervisor taskClient; private final SeekableStreamSupervisorSpec spec; private final SeekableStreamSupervisorIOConfig ioConfig; - private final Map autoscalerConfig; + private final AutoScalerConfig autoScalerConfig; private final SeekableStreamSupervisorTuningConfig tuningConfig; private final SeekableStreamIndexTaskTuningConfig taskTuningConfig; private final String supervisorId; @@ -614,9 +612,6 @@ boolean isValidTaskGroup(int taskGroupId, @Nullable TaskGroup taskGroup) private volatile boolean stopped = false; private volatile boolean lifecycleStarted = false; private final ServiceEmitter emitter; - private final boolean enableTaskAutoscaler; - private int taskCountMax; - private long minTriggerDynamicFrequency; public SeekableStreamSupervisor( final String supervisorId, @@ -640,18 +635,9 @@ public SeekableStreamSupervisor( this.useExclusiveStartingSequence = useExclusiveStartingSequence; this.dataSource = spec.getDataSchema().getDataSource(); this.ioConfig = spec.getIoConfig(); - this.autoscalerConfig = ioConfig.getAutoscalerConfig(); - log.debug("Get autoscalerConfig from IOConfig : [%s] in [%s]", autoscalerConfig, dataSource); - if (autoscalerConfig != null && !autoscalerConfig.isEmpty() && Boolean.parseBoolean(String.valueOf(autoscalerConfig.getOrDefault("enableTaskAutoscaler", true)))) { - log.info("enableTaskAutoscaler for datasource [%s]", dataSource); - this.taskCountMax = Integer.parseInt(String.valueOf(autoscalerConfig.getOrDefault("taskCountMax", TASK_COUNT_MAX))); - this.minTriggerDynamicFrequency = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("minTriggerDynamicFrequencyMillis", 600000))); - this.enableTaskAutoscaler = true; - } else { - log.info("Disable dynamic allocate tasks for [%s]", dataSource); - this.enableTaskAutoscaler = false; - } - + Map autoscalerConfigMap = ioConfig.getAutoscalerConfig(); + this.autoScalerConfig = mapper.convertValue(autoscalerConfigMap, AutoScalerConfig.class); + log.debug("Get autoscalerConfig from IOConfig : [%s] in [%s]", autoscalerConfigMap, dataSource); this.tuningConfig = spec.getTuningConfig(); this.taskTuningConfig = this.tuningConfig.convertToTaskTuningConfig(); this.supervisorId = supervisorId; @@ -665,14 +651,27 @@ public SeekableStreamSupervisor( ); int workerThreads; - if (enableTaskAutoscaler) { + int chatThreads; + if (autoscalerConfigMap != null && !autoscalerConfigMap.isEmpty() && autoScalerConfig.getEnableTaskAutoscaler()) { + log.info("enableTaskAutoscaler for datasource [%s]", dataSource); + workerThreads = (this.tuningConfig.getWorkerThreads() != null ? this.tuningConfig.getWorkerThreads() - : Math.min(10, taskCountMax)); + : Math.min(10, autoScalerConfig.getTaskCountMax())); + + chatThreads = (this.tuningConfig.getChatThreads() != null + ? this.tuningConfig.getChatThreads() + : Math.min(10, autoScalerConfig.getTaskCountMax() * this.ioConfig.getReplicas())); } else { + log.info("Disable dynamic allocate tasks for [%s]", dataSource); + workerThreads = (this.tuningConfig.getWorkerThreads() != null ? this.tuningConfig.getWorkerThreads() : Math.min(10, this.ioConfig.getTaskCount())); + + chatThreads = (this.tuningConfig.getChatThreads() != null + ? this.tuningConfig.getChatThreads() + : Math.min(10, this.ioConfig.getTaskCount() * this.ioConfig.getReplicas())); } this.workerExec = MoreExecutors.listeningDecorator( @@ -718,16 +717,6 @@ public Optional getTaskStatus(String id) tuningConfig.getChatRetries() * (tuningConfig.getHttpTimeout().getStandardSeconds() + IndexTaskClient.MAX_RETRY_WAIT_SECONDS) ); - int chatThreads; - if (enableTaskAutoscaler) { - chatThreads = (this.tuningConfig.getChatThreads() != null - ? this.tuningConfig.getChatThreads() - : Math.min(10, taskCountMax * this.ioConfig.getReplicas())); - } else { - chatThreads = (this.tuningConfig.getChatThreads() != null - ? this.tuningConfig.getChatThreads() - : Math.min(10, this.ioConfig.getTaskCount() * this.ioConfig.getReplicas())); - } this.taskClient = taskClientFactory.build( taskInfoProvider, diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java index d4b63eabf563..3ce1155eecf7 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java @@ -35,6 +35,7 @@ import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientFactory; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScaler; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DummyAutoScaler; import org.apache.druid.java.util.common.StringUtils; @@ -167,20 +168,26 @@ public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) { String dataSource = getId(); SupervisorTaskAutoscaler autoScaler = new DummyAutoScaler(supervisor, dataSource); - Map autoscalerConfig = ingestionSchema.getIOConfig().getAutoscalerConfig(); + Map autoscalerConfigMap = ingestionSchema.getIOConfig().getAutoscalerConfig(); + + // if autoscalerConfigMap is null then autoScalerConfig will be null + // if autoscalerConfigMap is empty then autoScalerConfig will be default values. + AutoScalerConfig autoScalerConfig = mapper.convertValue(autoscalerConfigMap, AutoScalerConfig.class); // kinesis'autoscalerConfig is always null for now, So that kinesis will hold a DummyAutoScaler. // only SeekableStreamSupervisor is supported here. - if (autoscalerConfig != null - && !autoscalerConfig.isEmpty() - && Boolean.parseBoolean(String.valueOf(autoscalerConfig.getOrDefault("enableTaskAutoscaler", true))) + if (autoscalerConfigMap != null + && !autoscalerConfigMap.isEmpty() + && autoScalerConfig.getEnableTaskAutoscaler() && supervisor instanceof SeekableStreamSupervisor) { - String autoScalerStrategy = String.valueOf(autoscalerConfig.getOrDefault("autoScalerStrategy", "default")); + String autoScalerStrategy = autoScalerConfig.getAutoScalerStrategy(); // will thorw 'Return value of String.hashCode() ignored : RV_RETURN_VALUE_IGNORED' just Suppress it. switch (StringUtils.toLowerCase(autoScalerStrategy)) { - default: autoScaler = new DefaultAutoScaler(supervisor, dataSource, autoscalerConfig, this); + default: { + autoScaler = new DefaultAutoScaler(supervisor, dataSource, autoScalerConfig, this); + } } } return autoScaler; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java new file mode 100644 index 000000000000..2e1727dcae58 --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.seekablestream.supervisor.autoscaler; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class AutoScalerConfig +{ + @JsonProperty("metricsCollectionIntervalMillis") + private long metricsCollectionIntervalMillis = 30000; + @JsonProperty("metricsCollectionRangeMillis") + private long metricsCollectionRangeMillis = 600000; + @JsonProperty("dynamicCheckStartDelayMillis") + private long dynamicCheckStartDelayMillis = 300000; + @JsonProperty("dynamicCheckPeriod") + private long dynamicCheckPeriod = 60000; + @JsonProperty("scaleOutThreshold") + private long scaleOutThreshold = 6000000; + @JsonProperty("scaleInThreshold") + private long scaleInThreshold = 1000000; + @JsonProperty("triggerScaleOutThresholdFrequency") + private double triggerScaleOutThresholdFrequency = 0.3; + @JsonProperty("triggerScaleInThresholdFrequency") + private double triggerScaleInThresholdFrequency = 0.9; + @JsonProperty("taskCountMax") + private int taskCountMax = 4; + @JsonProperty("taskCountMin") + private int taskCountMin = 1; + @JsonProperty("scaleInStep") + private int scaleInStep = 1; + @JsonProperty("scaleOutStep") + private int scaleOutStep = 2; + @JsonProperty("enableTaskAutoscaler") + private boolean enableTaskAutoscaler = true; + @JsonProperty("autoScalerStrategy") + private String autoScalerStrategy = "default"; + @JsonProperty("minTriggerDynamicFrequencyMillis") + private long minTriggerDynamicFrequencyMillis = 600000; + + + + @JsonProperty + public long getMetricsCollectionIntervalMillis() + { + return metricsCollectionIntervalMillis; + } + + @JsonProperty + public long getMetricsCollectionRangeMillis() + { + return metricsCollectionRangeMillis; + } + + @JsonProperty + public long getDynamicCheckStartDelayMillis() + { + return dynamicCheckStartDelayMillis; + } + + @JsonProperty + public long getDynamicCheckPeriod() + { + return dynamicCheckPeriod; + } + + @JsonProperty + public long getScaleOutThreshold() + { + return scaleOutThreshold; + } + + @JsonProperty + public long getScaleInThreshold() + { + return scaleInThreshold; + } + + @JsonProperty + public double getTriggerScaleOutThresholdFrequency() + { + return triggerScaleOutThresholdFrequency; + } + + @JsonProperty + public double getTriggerScaleInThresholdFrequency() + { + return triggerScaleInThresholdFrequency; + } + + @JsonProperty + public int getTaskCountMax() + { + return taskCountMax; + } + + @JsonProperty + public int getTaskCountMin() + { + return taskCountMin; + } + + @JsonProperty + public int getScaleInStep() + { + return scaleInStep; + } + + @JsonProperty + public int getScaleOutStep() + { + return scaleOutStep; + } + + @JsonProperty + public boolean getEnableTaskAutoscaler() + { + return enableTaskAutoscaler; + } + + @JsonProperty + public String getAutoScalerStrategy() + { + return autoScalerStrategy; + } + + @JsonProperty + public long getMinTriggerDynamicFrequencyMillis() + { + return minTriggerDynamicFrequencyMillis; + } + +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java index 4939b8de09ba..512a9cbb30de 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java @@ -31,7 +31,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -41,50 +40,28 @@ public class DefaultAutoScaler implements SupervisorTaskAutoscaler { private static final EmittingLogger log = new EmittingLogger(DefaultAutoScaler.class); private final String dataSource; - private final long metricsCollectionIntervalMillis; - private final long metricsCollectionRangeMillis; private final CircularFifoQueue lagMetricsQueue; - private final long dynamicCheckStartDelayMillis; - private final long dynamicCheckPeriod; - private final long scaleOutThreshold; - private final long scaleInThreshold; - private final double triggerScaleOutThresholdFrequency; - private final double triggerScaleInThresholdFrequency; - private final int taskCountMax; - private final int taskCountMin; - private final int scaleInStep; - private final int scaleOutStep; private final ScheduledExecutorService lagComputationExec; private final ScheduledExecutorService allocationExec; private final SupervisorSpec spec; private final SeekableStreamSupervisor supervisor; + private final AutoScalerConfig autoScalerConfig; private static ReentrantLock lock = new ReentrantLock(true); - public DefaultAutoScaler(Supervisor supervisor, String dataSource, Map autoscalerConfig, SupervisorSpec spec) + public DefaultAutoScaler(Supervisor supervisor, String dataSource, AutoScalerConfig autoScalerConfig, SupervisorSpec spec) { String supervisorId = StringUtils.format("KafkaSupervisor-%s", dataSource); this.dataSource = dataSource; - this.metricsCollectionIntervalMillis = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("metricsCollectionIntervalMillis", 30000))); - this.metricsCollectionRangeMillis = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("metricsCollectionRangeMillis", 600000))); - int slots = (int) (metricsCollectionRangeMillis / metricsCollectionIntervalMillis) + 1; - log.debug(" The interval of metrics collection is [%s], [%s] timeRange will collect [%s] data points for dataSource [%s].", metricsCollectionIntervalMillis, metricsCollectionRangeMillis, slots, dataSource); + int slots = (int) (autoScalerConfig.getMetricsCollectionRangeMillis() / autoScalerConfig.getMetricsCollectionIntervalMillis()) + 1; + log.debug(" The interval of metrics collection is [%s], [%s] timeRange will collect [%s] data points for dataSource [%s].", autoScalerConfig.getMetricsCollectionIntervalMillis(), autoScalerConfig.getMetricsCollectionRangeMillis(), slots, dataSource); this.lagMetricsQueue = new CircularFifoQueue<>(slots); - this.dynamicCheckStartDelayMillis = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("dynamicCheckStartDelayMillis", 300000))); - this.dynamicCheckPeriod = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("dynamicCheckPeriod", 60000))); - this.scaleOutThreshold = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("scaleOutThreshold", 6000000))); - this.scaleInThreshold = Long.parseLong(String.valueOf(autoscalerConfig.getOrDefault("scaleInThreshold", 1000000))); - this.triggerScaleOutThresholdFrequency = Double.parseDouble(String.valueOf(autoscalerConfig.getOrDefault("triggerScaleOutThresholdFrequency", 0.3))); - this.triggerScaleInThresholdFrequency = Double.parseDouble(String.valueOf(autoscalerConfig.getOrDefault("triggerScaleInThresholdFrequency", 0.9))); - this.taskCountMax = Integer.parseInt(String.valueOf(autoscalerConfig.getOrDefault("taskCountMax", SeekableStreamSupervisor.TASK_COUNT_MAX))); - this.taskCountMin = Integer.parseInt(String.valueOf(autoscalerConfig.getOrDefault("taskCountMin", SeekableStreamSupervisor.TASK_COUNT_MIN))); - this.scaleInStep = Integer.parseInt(String.valueOf(autoscalerConfig.getOrDefault("scaleInStep", 1))); - this.scaleOutStep = Integer.parseInt(String.valueOf(autoscalerConfig.getOrDefault("scaleOutStep", 2))); this.allocationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Allocation-%d"); this.lagComputationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Computation-%d"); this.spec = spec; this.supervisor = (SeekableStreamSupervisor) supervisor; + this.autoScalerConfig = autoScalerConfig; } @Override @@ -114,18 +91,18 @@ public Integer call() }; log.info("enableTaskAutoscaler for datasource [%s]", dataSource); - log.debug("Collect and compute lags at fixed rate of [%s] for dataSource[%s].", metricsCollectionIntervalMillis, dataSource); + log.debug("Collect and compute lags at fixed rate of [%s] for dataSource[%s].", autoScalerConfig.getMetricsCollectionIntervalMillis(), dataSource); lagComputationExec.scheduleAtFixedRate( collectAndComputeLags(), - dynamicCheckStartDelayMillis, // wait for tasks to start up - metricsCollectionIntervalMillis, + autoScalerConfig.getDynamicCheckStartDelayMillis(), // wait for tasks to start up + autoScalerConfig.getMetricsCollectionIntervalMillis(), TimeUnit.MILLISECONDS ); - log.debug("allocate task at fixed rate of [%s], dataSource [%s].", dynamicCheckPeriod, dataSource); + log.debug("allocate task at fixed rate of [%s], dataSource [%s].", autoScalerConfig.getDynamicCheckPeriod(), dataSource); allocationExec.scheduleAtFixedRate( supervisor.buildDynamicAllocationTask(scaleAction), - dynamicCheckStartDelayMillis + metricsCollectionRangeMillis, - dynamicCheckPeriod, + autoScalerConfig.getDynamicCheckStartDelayMillis() + autoScalerConfig.getMetricsCollectionRangeMillis(), + autoScalerConfig.getDynamicCheckPeriod(), TimeUnit.MILLISECONDS ); } @@ -212,16 +189,16 @@ private Integer computeDesireTaskCount(List lags) int within = 0; int metricsCount = lags.size(); for (Long lag : lags) { - if (lag >= scaleOutThreshold) { + if (lag >= autoScalerConfig.getScaleOutThreshold()) { beyond++; } - if (lag <= scaleInThreshold) { + if (lag <= autoScalerConfig.getScaleInThreshold()) { within++; } } double beyondProportion = beyond * 1.0 / metricsCount; double withinProportion = within * 1.0 / metricsCount; - log.debug("triggerScaleOutThresholdFrequency is [%s] and triggerScaleInThresholdFrequency is [%s] for dataSource [%s].", triggerScaleOutThresholdFrequency, triggerScaleInThresholdFrequency, dataSource); + log.debug("triggerScaleOutThresholdFrequency is [%s] and triggerScaleInThresholdFrequency is [%s] for dataSource [%s].", autoScalerConfig.getTriggerScaleOutThresholdFrequency(), autoScalerConfig.getTriggerScaleInThresholdFrequency(), dataSource); log.info("beyondProportion is [%s] and withinProportion is [%s] for dataSource [%s].", beyondProportion, withinProportion, dataSource); int currentActiveTaskCount = supervisor.getActiveTaskGroupsCount(); @@ -231,27 +208,27 @@ private Integer computeDesireTaskCount(List lags) } int desireActiveTaskCount; - if (beyondProportion >= triggerScaleOutThresholdFrequency) { + if (beyondProportion >= autoScalerConfig.getTriggerScaleOutThresholdFrequency()) { // Do Scale out - int taskCount = currentActiveTaskCount + scaleOutStep; - if (currentActiveTaskCount == taskCountMax) { + int taskCount = currentActiveTaskCount + autoScalerConfig.getScaleOutStep(); + if (currentActiveTaskCount == autoScalerConfig.getTaskCountMax()) { log.info("CurrentActiveTaskCount reach task count Max limit, skip to scale out tasks for dataSource [%s].", dataSource); return -1; } else { - desireActiveTaskCount = Math.min(taskCount, taskCountMax); + desireActiveTaskCount = Math.min(taskCount, autoScalerConfig.getTaskCountMax()); } return desireActiveTaskCount; } - if (withinProportion >= triggerScaleInThresholdFrequency) { + if (withinProportion >= autoScalerConfig.getTriggerScaleInThresholdFrequency()) { // Do Scale in - int taskCount = currentActiveTaskCount - scaleInStep; - if (currentActiveTaskCount == taskCountMin) { + int taskCount = currentActiveTaskCount - autoScalerConfig.getScaleInStep(); + if (currentActiveTaskCount == autoScalerConfig.getTaskCountMin()) { log.info("CurrentActiveTaskCount reach task count Min limit, skip to scale in tasks for dataSource [%s].", dataSource); return -1; } else { - desireActiveTaskCount = Math.max(taskCount, taskCountMin); + desireActiveTaskCount = Math.max(taskCount, autoScalerConfig.getTaskCountMin()); } log.debug("Start to scale in tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desireActiveTaskCount, dataSource); return desireActiveTaskCount; @@ -259,4 +236,9 @@ private Integer computeDesireTaskCount(List lags) return -1; } + + public AutoScalerConfig getAutoScalerConfig() + { + return autoScalerConfig; + } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 587c83f5cf30..41a74445d5f4 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -46,8 +46,10 @@ import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorReportPayload; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorSpec; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorTuningConfig; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScaler; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DummyAutoScaler; +import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.parsers.JSONPathSpec; import org.apache.druid.java.util.emitter.service.ServiceEmitter; @@ -119,7 +121,7 @@ public void setUp() spec = EasyMock.mock(SeekableStreamSupervisorSpec.class); supervisorConfig = new SupervisorStateManagerConfig(); indexTaskClientFactory = EasyMock.mock(SeekableStreamIndexTaskClientFactory.class); - mapper = EasyMock.mock(ObjectMapper.class); + mapper = new DefaultObjectMapper(); monitorSchedulerConfig = EasyMock.mock(DruidMonitorSchedulerConfig.class); supervisorStateManagerConfig = EasyMock.mock(SupervisorStateManagerConfig.class); supervisor4 = EasyMock.mock(SeekableStreamSupervisor.class); @@ -560,6 +562,56 @@ public void testAutoScalerCreated() SupervisorTaskAutoscaler autoscaler3 = spec.createAutoscaler(supervisor4); Assert.assertTrue(autoscaler3 instanceof DefaultAutoScaler); + } + + @Test + public void testDefaultAutoScalerConfigCreatedWithDefault() + { + EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes(); + EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes(); + EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); + EasyMock.replay(ingestionSchema); + + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(ImmutableMap.of("DummyKey", "DummyValue")).anyTimes(); + EasyMock.replay(seekableStreamSupervisorIOConfig); + + EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); + EasyMock.replay(supervisor4); + + TesstSeekableStreamSupervisorSpec spec = new TesstSeekableStreamSupervisorSpec(ingestionSchema, + null, + false, + taskStorage, + taskMaster, + indexerMetadataStorageCoordinator, + indexTaskClientFactory, + mapper, + emitter, + monitorSchedulerConfig, + rowIngestionMetersFactory, + supervisorStateManagerConfig, + supervisor4, + "id1"); + SupervisorTaskAutoscaler autoscaler = spec.createAutoscaler(supervisor4); + Assert.assertTrue(autoscaler instanceof DefaultAutoScaler); + DefaultAutoScaler defaultAutoScaler = (DefaultAutoScaler) autoscaler; + AutoScalerConfig autoScalerConfig = defaultAutoScaler.getAutoScalerConfig(); + Assert.assertEquals(autoScalerConfig.getMetricsCollectionIntervalMillis(), 30000); + Assert.assertEquals(autoScalerConfig.getMetricsCollectionRangeMillis(), 600000); + Assert.assertEquals(autoScalerConfig.getDynamicCheckStartDelayMillis(), 300000); + Assert.assertEquals(autoScalerConfig.getDynamicCheckPeriod(), 60000); + Assert.assertEquals(autoScalerConfig.getScaleOutThreshold(), 6000000); + Assert.assertEquals(autoScalerConfig.getScaleInThreshold(), 1000000); + Assert.assertEquals(autoScalerConfig.getTaskCountMax(), 4); + Assert.assertEquals(autoScalerConfig.getTaskCountMin(), 1); + Assert.assertEquals(autoScalerConfig.getScaleInStep(), 1); + Assert.assertEquals(autoScalerConfig.getScaleOutStep(), 2); + Assert.assertEquals(autoScalerConfig.getEnableTaskAutoscaler(), true); + Assert.assertEquals(autoScalerConfig.getAutoScalerStrategy(), "default"); + Assert.assertEquals(autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), 600000); + + + } @Test @@ -588,7 +640,7 @@ public void testSeekableStreamSupervisorSpecWithScaleOut() throws InterruptedExc LagStats lagStats = supervisor.computeLagStats(); - DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, getScaleOutProperties(), spec); + DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(), AutoScalerConfig.class), spec); supervisor.start(); autoScaler.start(); supervisor.runInternal(); @@ -626,7 +678,7 @@ public void testSeekableStreamSupervisorSpecWithScaleIn() throws InterruptedExce EasyMock.replay(taskMaster); TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); - DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, getScaleInProperties(), spec); + DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleInProperties(), AutoScalerConfig.class), spec); supervisor.start(); autoScaler.start(); supervisor.runInternal(); From f0c8d7877590876ac9e1e2277eea7b0cb810ec06 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Wed, 3 Feb 2021 22:27:30 +0800 Subject: [PATCH 31/47] add uts --- .../development/extensions-core/kafka-ingestion.md | 2 +- .../supervisor/SeekableStreamSupervisorSpec.java | 4 ++-- .../SeekableStreamSupervisorSpecTest.java | 14 ++++++++------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 5af4084ab460..83fa7b18421a 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -152,7 +152,7 @@ A sample supervisor spec is shown below: | Property | Description | Default | | ------------- | ------------- | ------------- | -| `enableTaskAutoscaler` | whether enable this feature or not. Set false here will disable `autoscaler` even though autoscalerConfig is not null| true | +| `enableTaskAutoscaler` | whether enable this feature or not. Set false here will disable `autoscaler` even though `autoscalerConfig` is not null| true | | `metricsCollectionIntervalMillis` | Define the frequency of lag points collection. | 30000 | | `metricsCollectionRangeMillis` | The total time window of lag collection, Use with `metricsCollectionIntervalMillis`，it means that in the recent `metricsCollectionRangeMill`, collect lag metric points every `metricsCollectionIntervalMillis`. | 600000 | | `scaleOutThreshold` | The Threshold of scale out action | 6000000 | diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java index 3ce1155eecf7..da4f9131175f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java @@ -170,8 +170,8 @@ public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) SupervisorTaskAutoscaler autoScaler = new DummyAutoScaler(supervisor, dataSource); Map autoscalerConfigMap = ingestionSchema.getIOConfig().getAutoscalerConfig(); - // if autoscalerConfigMap is null then autoScalerConfig will be null - // if autoscalerConfigMap is empty then autoScalerConfig will be default values. + // if autoscalerConfigMap is null then autoScalerConfig will be null. + // if autoscalerConfigMap is empty then autoScalerConfig will be default values and shouldn't be used. AutoScalerConfig autoScalerConfig = mapper.convertValue(autoscalerConfigMap, AutoScalerConfig.class); // kinesis'autoscalerConfig is always null for now, So that kinesis will hold a DummyAutoScaler. diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 41a74445d5f4..76db8d12f78f 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -562,6 +562,11 @@ public void testAutoScalerCreated() SupervisorTaskAutoscaler autoscaler3 = spec.createAutoscaler(supervisor4); Assert.assertTrue(autoscaler3 instanceof DefaultAutoScaler); + autoscalerConfig.clear(); + Assert.assertTrue(autoscalerConfig.isEmpty()); + SupervisorTaskAutoscaler autoscaler4 = spec.createAutoscaler(supervisor4); + Assert.assertTrue(autoscaler4 instanceof DummyAutoScaler); + } @Test @@ -572,7 +577,7 @@ public void testDefaultAutoScalerConfigCreatedWithDefault() EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(ImmutableMap.of("DummyKey", "DummyValue")).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(ImmutableMap.of("metricsCollectionIntervalMillis", "1")).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); @@ -596,7 +601,7 @@ public void testDefaultAutoScalerConfigCreatedWithDefault() Assert.assertTrue(autoscaler instanceof DefaultAutoScaler); DefaultAutoScaler defaultAutoScaler = (DefaultAutoScaler) autoscaler; AutoScalerConfig autoScalerConfig = defaultAutoScaler.getAutoScalerConfig(); - Assert.assertEquals(autoScalerConfig.getMetricsCollectionIntervalMillis(), 30000); + Assert.assertEquals(autoScalerConfig.getMetricsCollectionIntervalMillis(), 1); Assert.assertEquals(autoScalerConfig.getMetricsCollectionRangeMillis(), 600000); Assert.assertEquals(autoScalerConfig.getDynamicCheckStartDelayMillis(), 300000); Assert.assertEquals(autoScalerConfig.getDynamicCheckPeriod(), 60000); @@ -606,12 +611,9 @@ public void testDefaultAutoScalerConfigCreatedWithDefault() Assert.assertEquals(autoScalerConfig.getTaskCountMin(), 1); Assert.assertEquals(autoScalerConfig.getScaleInStep(), 1); Assert.assertEquals(autoScalerConfig.getScaleOutStep(), 2); - Assert.assertEquals(autoScalerConfig.getEnableTaskAutoscaler(), true); + Assert.assertTrue(autoScalerConfig.getEnableTaskAutoscaler()); Assert.assertEquals(autoScalerConfig.getAutoScalerStrategy(), "default"); Assert.assertEquals(autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), 600000); - - - } @Test From 0733590862fa43068e19b15d85dc5507732f7620 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 4 Feb 2021 01:22:19 +0800 Subject: [PATCH 32/47] use jackson to init AutoScalerConfig in IOConfig instead of Map<> --- .idea/misc.xml | 4 +- .../supervisor/KafkaSupervisorIOConfig.java | 4 +- .../kafka/supervisor/KafkaSupervisorTest.java | 93 ++++++++++++++++++- .../supervisor/KinesisSupervisorIOConfig.java | 5 +- .../supervisor/KinesisSupervisorTest.java | 9 +- .../supervisor/SeekableStreamSupervisor.java | 6 +- .../SeekableStreamSupervisorIOConfig.java | 9 +- .../SeekableStreamSupervisorSpec.java | 9 +- .../autoscaler/AutoScalerConfig.java | 86 ++++++++++------- .../SeekableStreamSupervisorSpecTest.java | 11 +-- .../SeekableStreamSupervisorStateTest.java | 3 +- 11 files changed, 175 insertions(+), 64 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java index 165fad7726f9..c96c0bb2fa2a 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java @@ -24,10 +24,12 @@ import com.google.common.base.Preconditions; import org.apache.druid.data.input.InputFormat; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorIOConfig; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.java.util.common.StringUtils; import org.joda.time.DateTime; import org.joda.time.Period; +import javax.annotation.Nullable; import java.util.Map; public class KafkaSupervisorIOConfig extends SeekableStreamSupervisorIOConfig @@ -51,7 +53,7 @@ public KafkaSupervisorIOConfig( @JsonProperty("taskCount") Integer taskCount, @JsonProperty("taskDuration") Period taskDuration, @JsonProperty("consumerProperties") Map consumerProperties, - @JsonProperty("autoscalerConfig") Map autoscalerConfig, + @Nullable @JsonProperty("autoscalerConfig") AutoScalerConfig autoscalerConfig, @JsonProperty("pollTimeout") Long pollTimeout, @JsonProperty("startDelay") Period startDelay, @JsonProperty("period") Period period, diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 65bb84a022e7..63eebbb93884 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -70,6 +70,7 @@ import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorSpec; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager; import org.apache.druid.indexing.seekablestream.supervisor.TaskReportData; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; @@ -287,7 +288,7 @@ public KafkaIndexTaskClient build( 1, new Period("PT1H"), consumerProperties, - autoscalerConfig, + OBJECT_MAPPER.convertValue(autoscalerConfig, AutoScalerConfig.class), KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), @@ -427,6 +428,96 @@ public KafkaIndexTaskClient build( autoscaler.stop(); } + + @Test + public void testKafkaSupervisorIOConfigInit() + { + final Map consumerProperties = KafkaConsumerConfigs.getConsumerProperties(); + consumerProperties.put("myCustomKey", "myCustomValue"); + consumerProperties.put("bootstrap.servers", kafkaHost); + + //autoscalerConfig = normal + KafkaSupervisorIOConfig kafkaSupervisorIOConfig = new KafkaSupervisorIOConfig( + topic, + INPUT_FORMAT, + 1, + 1, + new Period("PT1H"), + consumerProperties, + OBJECT_MAPPER.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", 1), AutoScalerConfig.class), + KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, + new Period("P1D"), + new Period("PT30S"), + true, + new Period("PT30M"), + null, + null, + null + ); + + AutoScalerConfig autoScalerConfig = kafkaSupervisorIOConfig.getAutoscalerConfig(); + Assert.assertNotNull(autoScalerConfig); + Assert.assertEquals(autoScalerConfig.getMetricsCollectionIntervalMillis(), 1); + Assert.assertEquals(autoScalerConfig.getMetricsCollectionRangeMillis(), 600000); + Assert.assertEquals(autoScalerConfig.getDynamicCheckStartDelayMillis(), 300000); + Assert.assertEquals(autoScalerConfig.getDynamicCheckPeriod(), 60000); + Assert.assertEquals(autoScalerConfig.getScaleOutThreshold(), 6000000); + Assert.assertEquals(autoScalerConfig.getScaleInThreshold(), 1000000); + Assert.assertEquals(autoScalerConfig.getTaskCountMax(), 4); + Assert.assertEquals(autoScalerConfig.getTaskCountMin(), 1); + Assert.assertEquals(autoScalerConfig.getScaleInStep(), 1); + Assert.assertEquals(autoScalerConfig.getScaleOutStep(), 2); + Assert.assertFalse(autoScalerConfig.getEnableTaskAutoscaler()); + Assert.assertEquals(autoScalerConfig.getAutoScalerStrategy(), "default"); + Assert.assertEquals(autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), 600000); + + // autoscalerConfig = null ; + KafkaSupervisorIOConfig kafkaSupervisorIOConfig2 = new KafkaSupervisorIOConfig( + topic, + INPUT_FORMAT, + 1, + 1, + new Period("PT1H"), + consumerProperties, + null, + KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, + new Period("P1D"), + new Period("PT30S"), + true, + new Period("PT30M"), + null, + null, + null + ); + + AutoScalerConfig autoScalerConfig2 = kafkaSupervisorIOConfig2.getAutoscalerConfig(); + Assert.assertNull(autoScalerConfig2); + + // autoscalerConfig = empty ; + KafkaSupervisorIOConfig kafkaSupervisorIOConfig3 = new KafkaSupervisorIOConfig( + topic, + INPUT_FORMAT, + 1, + 1, + new Period("PT1H"), + consumerProperties, + OBJECT_MAPPER.convertValue(new HashMap<>(), AutoScalerConfig.class), + KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, + new Period("P1D"), + new Period("PT30S"), + true, + new Period("PT30M"), + null, + null, + null + ); + + AutoScalerConfig autoScalerConfig3 = kafkaSupervisorIOConfig3.getAutoscalerConfig(); + Assert.assertFalse(autoScalerConfig3.getEnableTaskAutoscaler()); + } + + + @Test public void testCreateBaseTaskContexts() throws JsonProcessingException { diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java index 1460dc3d44e3..7db2cce72271 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java @@ -26,11 +26,10 @@ import org.apache.druid.indexing.kinesis.KinesisIndexTaskIOConfig; import org.apache.druid.indexing.kinesis.KinesisRegion; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorIOConfig; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.joda.time.DateTime; import org.joda.time.Period; -import java.util.Map; - public class KinesisSupervisorIOConfig extends SeekableStreamSupervisorIOConfig { private final String endpoint; @@ -72,7 +71,7 @@ public KinesisSupervisorIOConfig( @JsonProperty("fetchDelayMillis") Integer fetchDelayMillis, @JsonProperty("awsAssumedRoleArn") String awsAssumedRoleArn, @JsonProperty("awsExternalId") String awsExternalId, - @JsonProperty("autoscalerConfig") Map autoscalerConfig, + @JsonProperty("autoscalerConfig") AutoScalerConfig autoscalerConfig, @JsonProperty("deaggregate") boolean deaggregate ) { diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java index 2aaca0c4cf73..ec1c07c7c6ee 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java @@ -66,6 +66,7 @@ import org.apache.druid.indexing.seekablestream.common.StreamPartition; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager; import org.apache.druid.indexing.seekablestream.supervisor.TaskReportData; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; @@ -373,6 +374,8 @@ public void testKinesisIOConfig() null, false ); + AutoScalerConfig autoscalerConfig = kinesisSupervisorIOConfig.getAutoscalerConfig(); + Assert.assertNull(autoscalerConfig); } catch (Exception ex) { e = ex; @@ -399,7 +402,7 @@ public void testKinesisIOConfig() 1000, null, null, - new HashMap<>(), + OBJECT_MAPPER.convertValue(new HashMap<>(), AutoScalerConfig.class), false ); } @@ -408,6 +411,10 @@ public void testKinesisIOConfig() } Assert.assertNotNull(e); Assert.assertTrue(e instanceof UnsupportedOperationException); + + + + } @Test diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index eb2797cc5865..21ed86ab3122 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -635,9 +635,7 @@ public SeekableStreamSupervisor( this.useExclusiveStartingSequence = useExclusiveStartingSequence; this.dataSource = spec.getDataSchema().getDataSource(); this.ioConfig = spec.getIoConfig(); - Map autoscalerConfigMap = ioConfig.getAutoscalerConfig(); - this.autoScalerConfig = mapper.convertValue(autoscalerConfigMap, AutoScalerConfig.class); - log.debug("Get autoscalerConfig from IOConfig : [%s] in [%s]", autoscalerConfigMap, dataSource); + this.autoScalerConfig = ioConfig.getAutoscalerConfig(); this.tuningConfig = spec.getTuningConfig(); this.taskTuningConfig = this.tuningConfig.convertToTaskTuningConfig(); this.supervisorId = supervisorId; @@ -652,7 +650,7 @@ public SeekableStreamSupervisor( int workerThreads; int chatThreads; - if (autoscalerConfigMap != null && !autoscalerConfigMap.isEmpty() && autoScalerConfig.getEnableTaskAutoscaler()) { + if (autoScalerConfig != null && autoScalerConfig.getEnableTaskAutoscaler()) { log.info("enableTaskAutoscaler for datasource [%s]", dataSource); workerThreads = (this.tuningConfig.getWorkerThreads() != null diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java index 92af2be81852..ca25a16fe4ce 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java @@ -23,6 +23,7 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; import org.apache.druid.data.input.InputFormat; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.java.util.common.IAE; import org.joda.time.DateTime; import org.joda.time.Duration; @@ -30,8 +31,6 @@ import javax.annotation.Nullable; -import java.util.Map; - public abstract class SeekableStreamSupervisorIOConfig { @@ -48,7 +47,7 @@ public abstract class SeekableStreamSupervisorIOConfig private final Optional lateMessageRejectionPeriod; private final Optional earlyMessageRejectionPeriod; private final Optional lateMessageRejectionStartDateTime; - @Nullable private final Map autoscalerConfig; + @Nullable private final AutoScalerConfig autoscalerConfig; public SeekableStreamSupervisorIOConfig( String stream, @@ -62,7 +61,7 @@ public SeekableStreamSupervisorIOConfig( Period completionTimeout, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, - @Nullable Map autoscalerConfig, + @Nullable AutoScalerConfig autoscalerConfig, DateTime lateMessageRejectionStartDateTime ) { @@ -121,7 +120,7 @@ public Integer getReplicas() @Nullable @JsonProperty - public Map getAutoscalerConfig() + public AutoScalerConfig getAutoscalerConfig() { return autoscalerConfig; } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java index da4f9131175f..f4c9417fb953 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java @@ -168,16 +168,11 @@ public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) { String dataSource = getId(); SupervisorTaskAutoscaler autoScaler = new DummyAutoScaler(supervisor, dataSource); - Map autoscalerConfigMap = ingestionSchema.getIOConfig().getAutoscalerConfig(); - - // if autoscalerConfigMap is null then autoScalerConfig will be null. - // if autoscalerConfigMap is empty then autoScalerConfig will be default values and shouldn't be used. - AutoScalerConfig autoScalerConfig = mapper.convertValue(autoscalerConfigMap, AutoScalerConfig.class); + AutoScalerConfig autoScalerConfig = ingestionSchema.getIOConfig().getAutoscalerConfig(); // kinesis'autoscalerConfig is always null for now, So that kinesis will hold a DummyAutoScaler. // only SeekableStreamSupervisor is supported here. - if (autoscalerConfigMap != null - && !autoscalerConfigMap.isEmpty() + if (autoScalerConfig != null && autoScalerConfig.getEnableTaskAutoscaler() && supervisor instanceof SeekableStreamSupervisor) { diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java index 2e1727dcae58..add6b87a8501 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java @@ -19,42 +19,63 @@ package org.apache.druid.indexing.seekablestream.supervisor.autoscaler; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import javax.annotation.Nullable; + public class AutoScalerConfig { - @JsonProperty("metricsCollectionIntervalMillis") - private long metricsCollectionIntervalMillis = 30000; - @JsonProperty("metricsCollectionRangeMillis") - private long metricsCollectionRangeMillis = 600000; - @JsonProperty("dynamicCheckStartDelayMillis") - private long dynamicCheckStartDelayMillis = 300000; - @JsonProperty("dynamicCheckPeriod") - private long dynamicCheckPeriod = 60000; - @JsonProperty("scaleOutThreshold") - private long scaleOutThreshold = 6000000; - @JsonProperty("scaleInThreshold") - private long scaleInThreshold = 1000000; - @JsonProperty("triggerScaleOutThresholdFrequency") - private double triggerScaleOutThresholdFrequency = 0.3; - @JsonProperty("triggerScaleInThresholdFrequency") - private double triggerScaleInThresholdFrequency = 0.9; - @JsonProperty("taskCountMax") - private int taskCountMax = 4; - @JsonProperty("taskCountMin") - private int taskCountMin = 1; - @JsonProperty("scaleInStep") - private int scaleInStep = 1; - @JsonProperty("scaleOutStep") - private int scaleOutStep = 2; - @JsonProperty("enableTaskAutoscaler") - private boolean enableTaskAutoscaler = true; - @JsonProperty("autoScalerStrategy") - private String autoScalerStrategy = "default"; - @JsonProperty("minTriggerDynamicFrequencyMillis") - private long minTriggerDynamicFrequencyMillis = 600000; - - + private final long metricsCollectionIntervalMillis; + private final long metricsCollectionRangeMillis; + private final long dynamicCheckStartDelayMillis; + private final long dynamicCheckPeriod; + private final long scaleOutThreshold; + private final long scaleInThreshold; + private final double triggerScaleOutThresholdFrequency; + private final double triggerScaleInThresholdFrequency; + private final int taskCountMax; + private final int taskCountMin; + private final int scaleInStep; + private final int scaleOutStep; + private final boolean enableTaskAutoscaler; + private final String autoScalerStrategy; + private final long minTriggerDynamicFrequencyMillis; + + @JsonCreator + public AutoScalerConfig( + @Nullable @JsonProperty("metricsCollectionIntervalMillis") Long metricsCollectionIntervalMillis, + @Nullable @JsonProperty("metricsCollectionRangeMillis") Long metricsCollectionRangeMillis, + @Nullable @JsonProperty("dynamicCheckStartDelayMillis") Long dynamicCheckStartDelayMillis, + @Nullable @JsonProperty("dynamicCheckPeriod") Long dynamicCheckPeriod, + @Nullable @JsonProperty("scaleOutThreshold") Long scaleOutThreshold, + @Nullable @JsonProperty("scaleInThreshold") Long scaleInThreshold, + @Nullable @JsonProperty("triggerScaleOutThresholdFrequency") Double triggerScaleOutThresholdFrequency, + @Nullable @JsonProperty("triggerScaleInThresholdFrequency") Double triggerScaleInThresholdFrequency, + @Nullable @JsonProperty("taskCountMax") Integer taskCountMax, + @Nullable @JsonProperty("taskCountMin") Integer taskCountMin, + @Nullable @JsonProperty("scaleInStep") Integer scaleInStep, + @Nullable @JsonProperty("scaleOutStep") Integer scaleOutStep, + @Nullable @JsonProperty("enableTaskAutoscaler") Boolean enableTaskAutoscaler, + @Nullable @JsonProperty("autoScalerStrategy") String autoScalerStrategy, + @Nullable @JsonProperty("minTriggerDynamicFrequencyMillis") Long minTriggerDynamicFrequencyMillis) + { + this.metricsCollectionIntervalMillis = metricsCollectionIntervalMillis != null ? metricsCollectionIntervalMillis : 30000; + this.metricsCollectionRangeMillis = metricsCollectionRangeMillis != null ? metricsCollectionRangeMillis : 600000; + this.dynamicCheckStartDelayMillis = dynamicCheckStartDelayMillis != null ? dynamicCheckStartDelayMillis : 300000; + this.dynamicCheckPeriod = dynamicCheckPeriod != null ? dynamicCheckPeriod : 60000; + this.scaleOutThreshold = scaleOutThreshold != null ? scaleOutThreshold : 6000000; + this.scaleInThreshold = scaleInThreshold != null ? scaleInThreshold : 1000000; + this.triggerScaleOutThresholdFrequency = triggerScaleOutThresholdFrequency != null ? triggerScaleOutThresholdFrequency : 0.3; + this.triggerScaleInThresholdFrequency = triggerScaleInThresholdFrequency != null ? triggerScaleInThresholdFrequency : 0.9; + this.taskCountMax = taskCountMax != null ? taskCountMax : 4; + this.taskCountMin = taskCountMin != null ? taskCountMin : 1; + this.scaleInStep = scaleInStep != null ? scaleInStep : 1; + this.scaleOutStep = scaleOutStep != null ? scaleOutStep : 2; + this.enableTaskAutoscaler = enableTaskAutoscaler != null ? enableTaskAutoscaler : false; + this.autoScalerStrategy = autoScalerStrategy != null ? autoScalerStrategy : "default"; + this.minTriggerDynamicFrequencyMillis = minTriggerDynamicFrequencyMillis != null ? minTriggerDynamicFrequencyMillis : 600000; + } @JsonProperty public long getMetricsCollectionIntervalMillis() @@ -145,5 +166,4 @@ public long getMinTriggerDynamicFrequencyMillis() { return minTriggerDynamicFrequencyMillis; } - } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 76db8d12f78f..cfcb013b71fb 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -531,7 +531,7 @@ public void testAutoScalerCreated() EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(autoscalerConfig).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, AutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); @@ -577,7 +577,7 @@ public void testDefaultAutoScalerConfigCreatedWithDefault() EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(ImmutableMap.of("metricsCollectionIntervalMillis", "1")).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", "1", "enableTaskAutoscaler", true), AutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); @@ -611,7 +611,6 @@ public void testDefaultAutoScalerConfigCreatedWithDefault() Assert.assertEquals(autoScalerConfig.getTaskCountMin(), 1); Assert.assertEquals(autoScalerConfig.getScaleInStep(), 1); Assert.assertEquals(autoScalerConfig.getScaleOutStep(), 2); - Assert.assertTrue(autoScalerConfig.getEnableTaskAutoscaler()); Assert.assertEquals(autoScalerConfig.getAutoScalerStrategy(), "default"); Assert.assertEquals(autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), 600000); } @@ -768,7 +767,7 @@ private static DataSchema getDataSchema() ); } - private static SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boolean scaleOut) + private SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boolean scaleOut) { if (scaleOut) { return new SeekableStreamSupervisorIOConfig( @@ -782,7 +781,7 @@ private static SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boole false, new Period("PT30M"), null, - null, getScaleOutProperties(), null + null, mapper.convertValue(getScaleOutProperties(), AutoScalerConfig.class), null ) {}; } else { return new SeekableStreamSupervisorIOConfig( @@ -796,7 +795,7 @@ null, getScaleOutProperties(), null false, new Period("PT30M"), null, - null, getScaleInProperties(), null + null, mapper.convertValue(getScaleInProperties(), AutoScalerConfig.class), null ) {}; } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index 2e15f0485063..58dfcae1c54c 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -59,6 +59,7 @@ import org.apache.druid.indexing.seekablestream.common.StreamPartition; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager.SeekableStreamExceptionEvent; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager.SeekableStreamState; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; @@ -829,7 +830,7 @@ private static SeekableStreamSupervisorIOConfig getIOConfig() false, new Period("PT30M"), null, - null, getProperties(), null + null, OBJECT_MAPPER.convertValue(getProperties(), AutoScalerConfig.class), null ) { }; From 972690294bf199ec0b93d01d5f44e3d9c008720c Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 4 Feb 2021 12:18:32 +0800 Subject: [PATCH 33/47] autoscalerConfig interface and provide a defaultAutoScalerConfig --- .idea/misc.xml | 4 +- .../extensions-core/kafka-ingestion.md | 4 +- .../kafka/supervisor/KafkaSupervisorTest.java | 36 ++-- .../supervisor/KinesisSupervisorTest.java | 7 +- .../supervisor/SeekableStreamSupervisor.java | 2 +- .../autoscaler/AutoScalerConfig.java | 163 ++-------------- .../autoscaler/DefaultAutoScaler.java | 46 ++--- .../autoscaler/DefaultAutoScalerConfig.java | 174 ++++++++++++++++++ .../SeekableStreamSupervisorSpecTest.java | 40 ++-- .../SeekableStreamSupervisorStateTest.java | 4 +- 10 files changed, 265 insertions(+), 215 deletions(-) create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScalerConfig.java diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 83fa7b18421a..619df3974fed 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -146,13 +146,13 @@ A sample supervisor spec is shown below: |`lateMessageRejectionStartDateTime`|ISO8601 DateTime|Configure tasks to reject messages with timestamps earlier than this date time; for example if this is set to `2016-01-01T11:00Z` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline).|no (default == none)| |`lateMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps earlier than this period before the task was created; for example if this is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline). Please note that only one of `lateMessageRejectionPeriod` or `lateMessageRejectionStartDateTime` can be specified.|no (default == none)| |`earlyMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps later than this period after the task reached its taskDuration; for example if this is set to `PT1H`, the taskDuration is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps later than *2016-01-01T14:00Z* will be dropped. **Note:** Tasks sometimes run past their task duration, for example, in cases of supervisor failover. Setting earlyMessageRejectionPeriod too low may cause messages to be dropped unexpectedly whenever a task runs past its originally configured task duration.|no (default == none)| -|`autoscalerConfig`|Object|`autoscalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for Kafka indexing as of now. See [Tasks Autoscaler Properties](#Tasks Autoscaler Properties) for details.|no (default == null)| +|`autoscalerConfigDefault`|Object|`autoscalerConfigDefault` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for Kafka indexing as of now. See [Tasks Autoscaler Properties](#Tasks Autoscaler Properties) for details.|no (default == null)| #### Tasks Autoscaler Properties | Property | Description | Default | | ------------- | ------------- | ------------- | -| `enableTaskAutoscaler` | whether enable this feature or not. Set false here will disable `autoscaler` even though `autoscalerConfig` is not null| true | +| `enableTaskAutoscaler` | whether enable this feature or not. Set false here will disable `autoscaler` even though `autoscalerConfigDefault` is not null| true | | `metricsCollectionIntervalMillis` | Define the frequency of lag points collection. | 30000 | | `metricsCollectionRangeMillis` | The total time window of lag collection, Use with `metricsCollectionIntervalMillis`，it means that in the recent `metricsCollectionRangeMill`, collect lag metric points every `metricsCollectionIntervalMillis`. | 600000 | | `scaleOutThreshold` | The Threshold of scale out action | 6000000 | diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 63eebbb93884..fda3f144a2b0 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -71,6 +71,7 @@ import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager; import org.apache.druid.indexing.seekablestream.supervisor.TaskReportData; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScalerConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; @@ -288,7 +289,7 @@ public KafkaIndexTaskClient build( 1, new Period("PT1H"), consumerProperties, - OBJECT_MAPPER.convertValue(autoscalerConfig, AutoScalerConfig.class), + OBJECT_MAPPER.convertValue(autoscalerConfig, DefaultAutoScalerConfig.class), KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), @@ -444,7 +445,7 @@ public void testKafkaSupervisorIOConfigInit() 1, new Period("PT1H"), consumerProperties, - OBJECT_MAPPER.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", 1), AutoScalerConfig.class), + OBJECT_MAPPER.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", 1), DefaultAutoScalerConfig.class), KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), @@ -457,19 +458,21 @@ public void testKafkaSupervisorIOConfigInit() AutoScalerConfig autoScalerConfig = kafkaSupervisorIOConfig.getAutoscalerConfig(); Assert.assertNotNull(autoScalerConfig); - Assert.assertEquals(autoScalerConfig.getMetricsCollectionIntervalMillis(), 1); - Assert.assertEquals(autoScalerConfig.getMetricsCollectionRangeMillis(), 600000); - Assert.assertEquals(autoScalerConfig.getDynamicCheckStartDelayMillis(), 300000); - Assert.assertEquals(autoScalerConfig.getDynamicCheckPeriod(), 60000); - Assert.assertEquals(autoScalerConfig.getScaleOutThreshold(), 6000000); - Assert.assertEquals(autoScalerConfig.getScaleInThreshold(), 1000000); - Assert.assertEquals(autoScalerConfig.getTaskCountMax(), 4); - Assert.assertEquals(autoScalerConfig.getTaskCountMin(), 1); - Assert.assertEquals(autoScalerConfig.getScaleInStep(), 1); - Assert.assertEquals(autoScalerConfig.getScaleOutStep(), 2); - Assert.assertFalse(autoScalerConfig.getEnableTaskAutoscaler()); - Assert.assertEquals(autoScalerConfig.getAutoScalerStrategy(), "default"); - Assert.assertEquals(autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), 600000); + Assert.assertTrue(autoScalerConfig instanceof DefaultAutoScalerConfig); + DefaultAutoScalerConfig defaultAutoScalerConfig = (DefaultAutoScalerConfig) autoScalerConfig; + Assert.assertEquals(defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), 1); + Assert.assertEquals(defaultAutoScalerConfig.getMetricsCollectionRangeMillis(), 600000); + Assert.assertEquals(defaultAutoScalerConfig.getDynamicCheckStartDelayMillis(), 300000); + Assert.assertEquals(defaultAutoScalerConfig.getDynamicCheckPeriod(), 60000); + Assert.assertEquals(defaultAutoScalerConfig.getScaleOutThreshold(), 6000000); + Assert.assertEquals(defaultAutoScalerConfig.getScaleInThreshold(), 1000000); + Assert.assertEquals(defaultAutoScalerConfig.getTaskCountMax(), 4); + Assert.assertEquals(defaultAutoScalerConfig.getTaskCountMin(), 1); + Assert.assertEquals(defaultAutoScalerConfig.getScaleInStep(), 1); + Assert.assertEquals(defaultAutoScalerConfig.getScaleOutStep(), 2); + Assert.assertFalse(defaultAutoScalerConfig.getEnableTaskAutoscaler()); + Assert.assertEquals(defaultAutoScalerConfig.getAutoScalerStrategy(), "default"); + Assert.assertEquals(defaultAutoScalerConfig.getMinTriggerDynamicFrequencyMillis(), 600000); // autoscalerConfig = null ; KafkaSupervisorIOConfig kafkaSupervisorIOConfig2 = new KafkaSupervisorIOConfig( @@ -501,7 +504,7 @@ public void testKafkaSupervisorIOConfigInit() 1, new Period("PT1H"), consumerProperties, - OBJECT_MAPPER.convertValue(new HashMap<>(), AutoScalerConfig.class), + OBJECT_MAPPER.convertValue(new HashMap<>(), DefaultAutoScalerConfig.class), KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), @@ -513,6 +516,7 @@ public void testKafkaSupervisorIOConfigInit() ); AutoScalerConfig autoScalerConfig3 = kafkaSupervisorIOConfig3.getAutoscalerConfig(); + Assert.assertTrue(autoScalerConfig3 instanceof DefaultAutoScalerConfig); Assert.assertFalse(autoScalerConfig3.getEnableTaskAutoscaler()); } diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java index ec1c07c7c6ee..4127364ea724 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java @@ -67,6 +67,7 @@ import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager; import org.apache.druid.indexing.seekablestream.supervisor.TaskReportData; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScalerConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; @@ -374,8 +375,8 @@ public void testKinesisIOConfig() null, false ); - AutoScalerConfig autoscalerConfig = kinesisSupervisorIOConfig.getAutoscalerConfig(); - Assert.assertNull(autoscalerConfig); + AutoScalerConfig autoscalerConfigDefault = kinesisSupervisorIOConfig.getAutoscalerConfig(); + Assert.assertNull(autoscalerConfigDefault); } catch (Exception ex) { e = ex; @@ -402,7 +403,7 @@ public void testKinesisIOConfig() 1000, null, null, - OBJECT_MAPPER.convertValue(new HashMap<>(), AutoScalerConfig.class), + OBJECT_MAPPER.convertValue(new HashMap<>(), DefaultAutoScalerConfig.class), false ); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 21ed86ab3122..aa1636b3cec0 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -354,7 +354,7 @@ public void handle() return; } } - if (nowTime - dynamicTriggerLastRunTime < autoScalerConfig.getMinTriggerDynamicFrequencyMillis()) { + if (autoScalerConfig != null && nowTime - dynamicTriggerLastRunTime < autoScalerConfig.getMinTriggerDynamicFrequencyMillis()) { log.info("NowTime - dynamicTriggerLastRunTime is [%s]. Defined minTriggerDynamicFrequency is [%s] for dataSource [%s], CLAM DOWN NOW !", nowTime - dynamicTriggerLastRunTime, autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), dataSource); return; } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java index add6b87a8501..ca3553dda28d 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java @@ -19,151 +19,22 @@ package org.apache.druid.indexing.seekablestream.supervisor.autoscaler; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; - -import javax.annotation.Nullable; - -public class AutoScalerConfig +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.apache.druid.guice.annotations.UnstableApi; + +@UnstableApi +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "autoScalerStrategy", defaultImpl = DefaultAutoScalerConfig.class) +@JsonSubTypes(value = { + @Type(name = "default", value = DefaultAutoScalerConfig.class) +}) +public interface AutoScalerConfig { - private final long metricsCollectionIntervalMillis; - private final long metricsCollectionRangeMillis; - private final long dynamicCheckStartDelayMillis; - private final long dynamicCheckPeriod; - private final long scaleOutThreshold; - private final long scaleInThreshold; - private final double triggerScaleOutThresholdFrequency; - private final double triggerScaleInThresholdFrequency; - private final int taskCountMax; - private final int taskCountMin; - private final int scaleInStep; - private final int scaleOutStep; - private final boolean enableTaskAutoscaler; - private final String autoScalerStrategy; - private final long minTriggerDynamicFrequencyMillis; - - @JsonCreator - public AutoScalerConfig( - @Nullable @JsonProperty("metricsCollectionIntervalMillis") Long metricsCollectionIntervalMillis, - @Nullable @JsonProperty("metricsCollectionRangeMillis") Long metricsCollectionRangeMillis, - @Nullable @JsonProperty("dynamicCheckStartDelayMillis") Long dynamicCheckStartDelayMillis, - @Nullable @JsonProperty("dynamicCheckPeriod") Long dynamicCheckPeriod, - @Nullable @JsonProperty("scaleOutThreshold") Long scaleOutThreshold, - @Nullable @JsonProperty("scaleInThreshold") Long scaleInThreshold, - @Nullable @JsonProperty("triggerScaleOutThresholdFrequency") Double triggerScaleOutThresholdFrequency, - @Nullable @JsonProperty("triggerScaleInThresholdFrequency") Double triggerScaleInThresholdFrequency, - @Nullable @JsonProperty("taskCountMax") Integer taskCountMax, - @Nullable @JsonProperty("taskCountMin") Integer taskCountMin, - @Nullable @JsonProperty("scaleInStep") Integer scaleInStep, - @Nullable @JsonProperty("scaleOutStep") Integer scaleOutStep, - @Nullable @JsonProperty("enableTaskAutoscaler") Boolean enableTaskAutoscaler, - @Nullable @JsonProperty("autoScalerStrategy") String autoScalerStrategy, - @Nullable @JsonProperty("minTriggerDynamicFrequencyMillis") Long minTriggerDynamicFrequencyMillis) - { - this.metricsCollectionIntervalMillis = metricsCollectionIntervalMillis != null ? metricsCollectionIntervalMillis : 30000; - this.metricsCollectionRangeMillis = metricsCollectionRangeMillis != null ? metricsCollectionRangeMillis : 600000; - this.dynamicCheckStartDelayMillis = dynamicCheckStartDelayMillis != null ? dynamicCheckStartDelayMillis : 300000; - this.dynamicCheckPeriod = dynamicCheckPeriod != null ? dynamicCheckPeriod : 60000; - this.scaleOutThreshold = scaleOutThreshold != null ? scaleOutThreshold : 6000000; - this.scaleInThreshold = scaleInThreshold != null ? scaleInThreshold : 1000000; - this.triggerScaleOutThresholdFrequency = triggerScaleOutThresholdFrequency != null ? triggerScaleOutThresholdFrequency : 0.3; - this.triggerScaleInThresholdFrequency = triggerScaleInThresholdFrequency != null ? triggerScaleInThresholdFrequency : 0.9; - this.taskCountMax = taskCountMax != null ? taskCountMax : 4; - this.taskCountMin = taskCountMin != null ? taskCountMin : 1; - this.scaleInStep = scaleInStep != null ? scaleInStep : 1; - this.scaleOutStep = scaleOutStep != null ? scaleOutStep : 2; - this.enableTaskAutoscaler = enableTaskAutoscaler != null ? enableTaskAutoscaler : false; - this.autoScalerStrategy = autoScalerStrategy != null ? autoScalerStrategy : "default"; - this.minTriggerDynamicFrequencyMillis = minTriggerDynamicFrequencyMillis != null ? minTriggerDynamicFrequencyMillis : 600000; - } - - @JsonProperty - public long getMetricsCollectionIntervalMillis() - { - return metricsCollectionIntervalMillis; - } - - @JsonProperty - public long getMetricsCollectionRangeMillis() - { - return metricsCollectionRangeMillis; - } - - @JsonProperty - public long getDynamicCheckStartDelayMillis() - { - return dynamicCheckStartDelayMillis; - } - - @JsonProperty - public long getDynamicCheckPeriod() - { - return dynamicCheckPeriod; - } - - @JsonProperty - public long getScaleOutThreshold() - { - return scaleOutThreshold; - } - - @JsonProperty - public long getScaleInThreshold() - { - return scaleInThreshold; - } - - @JsonProperty - public double getTriggerScaleOutThresholdFrequency() - { - return triggerScaleOutThresholdFrequency; - } - - @JsonProperty - public double getTriggerScaleInThresholdFrequency() - { - return triggerScaleInThresholdFrequency; - } - - @JsonProperty - public int getTaskCountMax() - { - return taskCountMax; - } - - @JsonProperty - public int getTaskCountMin() - { - return taskCountMin; - } - - @JsonProperty - public int getScaleInStep() - { - return scaleInStep; - } - - @JsonProperty - public int getScaleOutStep() - { - return scaleOutStep; - } - - @JsonProperty - public boolean getEnableTaskAutoscaler() - { - return enableTaskAutoscaler; - } - - @JsonProperty - public String getAutoScalerStrategy() - { - return autoScalerStrategy; - } - - @JsonProperty - public long getMinTriggerDynamicFrequencyMillis() - { - return minTriggerDynamicFrequencyMillis; - } + boolean getEnableTaskAutoscaler(); + long getMinTriggerDynamicFrequencyMillis(); + String getAutoScalerStrategy(); + int getTaskCountMax(); + int getTaskCountMin(); } + diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java index 512a9cbb30de..82bf3b1d4854 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java @@ -45,23 +45,23 @@ public class DefaultAutoScaler implements SupervisorTaskAutoscaler private final ScheduledExecutorService allocationExec; private final SupervisorSpec spec; private final SeekableStreamSupervisor supervisor; - private final AutoScalerConfig autoScalerConfig; + private final DefaultAutoScalerConfig defaultAutoScalerConfig; private static ReentrantLock lock = new ReentrantLock(true); public DefaultAutoScaler(Supervisor supervisor, String dataSource, AutoScalerConfig autoScalerConfig, SupervisorSpec spec) { + this.defaultAutoScalerConfig = (DefaultAutoScalerConfig) autoScalerConfig; String supervisorId = StringUtils.format("KafkaSupervisor-%s", dataSource); this.dataSource = dataSource; - int slots = (int) (autoScalerConfig.getMetricsCollectionRangeMillis() / autoScalerConfig.getMetricsCollectionIntervalMillis()) + 1; - log.debug(" The interval of metrics collection is [%s], [%s] timeRange will collect [%s] data points for dataSource [%s].", autoScalerConfig.getMetricsCollectionIntervalMillis(), autoScalerConfig.getMetricsCollectionRangeMillis(), slots, dataSource); + int slots = (int) (defaultAutoScalerConfig.getMetricsCollectionRangeMillis() / defaultAutoScalerConfig.getMetricsCollectionIntervalMillis()) + 1; + log.debug(" The interval of metrics collection is [%s], [%s] timeRange will collect [%s] data points for dataSource [%s].", defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), defaultAutoScalerConfig.getMetricsCollectionRangeMillis(), slots, dataSource); this.lagMetricsQueue = new CircularFifoQueue<>(slots); this.allocationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Allocation-%d"); this.lagComputationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Computation-%d"); this.spec = spec; this.supervisor = (SeekableStreamSupervisor) supervisor; - this.autoScalerConfig = autoScalerConfig; } @Override @@ -91,18 +91,18 @@ public Integer call() }; log.info("enableTaskAutoscaler for datasource [%s]", dataSource); - log.debug("Collect and compute lags at fixed rate of [%s] for dataSource[%s].", autoScalerConfig.getMetricsCollectionIntervalMillis(), dataSource); + log.debug("Collect and compute lags at fixed rate of [%s] for dataSource[%s].", defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), dataSource); lagComputationExec.scheduleAtFixedRate( collectAndComputeLags(), - autoScalerConfig.getDynamicCheckStartDelayMillis(), // wait for tasks to start up - autoScalerConfig.getMetricsCollectionIntervalMillis(), + defaultAutoScalerConfig.getDynamicCheckStartDelayMillis(), // wait for tasks to start up + defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), TimeUnit.MILLISECONDS ); - log.debug("allocate task at fixed rate of [%s], dataSource [%s].", autoScalerConfig.getDynamicCheckPeriod(), dataSource); + log.debug("allocate task at fixed rate of [%s], dataSource [%s].", defaultAutoScalerConfig.getDynamicCheckPeriod(), dataSource); allocationExec.scheduleAtFixedRate( supervisor.buildDynamicAllocationTask(scaleAction), - autoScalerConfig.getDynamicCheckStartDelayMillis() + autoScalerConfig.getMetricsCollectionRangeMillis(), - autoScalerConfig.getDynamicCheckPeriod(), + defaultAutoScalerConfig.getDynamicCheckStartDelayMillis() + defaultAutoScalerConfig.getMetricsCollectionRangeMillis(), + defaultAutoScalerConfig.getDynamicCheckPeriod(), TimeUnit.MILLISECONDS ); } @@ -189,16 +189,16 @@ private Integer computeDesireTaskCount(List lags) int within = 0; int metricsCount = lags.size(); for (Long lag : lags) { - if (lag >= autoScalerConfig.getScaleOutThreshold()) { + if (lag >= defaultAutoScalerConfig.getScaleOutThreshold()) { beyond++; } - if (lag <= autoScalerConfig.getScaleInThreshold()) { + if (lag <= defaultAutoScalerConfig.getScaleInThreshold()) { within++; } } double beyondProportion = beyond * 1.0 / metricsCount; double withinProportion = within * 1.0 / metricsCount; - log.debug("triggerScaleOutThresholdFrequency is [%s] and triggerScaleInThresholdFrequency is [%s] for dataSource [%s].", autoScalerConfig.getTriggerScaleOutThresholdFrequency(), autoScalerConfig.getTriggerScaleInThresholdFrequency(), dataSource); + log.debug("triggerScaleOutThresholdFrequency is [%s] and triggerScaleInThresholdFrequency is [%s] for dataSource [%s].", defaultAutoScalerConfig.getTriggerScaleOutThresholdFrequency(), defaultAutoScalerConfig.getTriggerScaleInThresholdFrequency(), dataSource); log.info("beyondProportion is [%s] and withinProportion is [%s] for dataSource [%s].", beyondProportion, withinProportion, dataSource); int currentActiveTaskCount = supervisor.getActiveTaskGroupsCount(); @@ -208,27 +208,27 @@ private Integer computeDesireTaskCount(List lags) } int desireActiveTaskCount; - if (beyondProportion >= autoScalerConfig.getTriggerScaleOutThresholdFrequency()) { + if (beyondProportion >= defaultAutoScalerConfig.getTriggerScaleOutThresholdFrequency()) { // Do Scale out - int taskCount = currentActiveTaskCount + autoScalerConfig.getScaleOutStep(); - if (currentActiveTaskCount == autoScalerConfig.getTaskCountMax()) { + int taskCount = currentActiveTaskCount + defaultAutoScalerConfig.getScaleOutStep(); + if (currentActiveTaskCount == defaultAutoScalerConfig.getTaskCountMax()) { log.info("CurrentActiveTaskCount reach task count Max limit, skip to scale out tasks for dataSource [%s].", dataSource); return -1; } else { - desireActiveTaskCount = Math.min(taskCount, autoScalerConfig.getTaskCountMax()); + desireActiveTaskCount = Math.min(taskCount, defaultAutoScalerConfig.getTaskCountMax()); } return desireActiveTaskCount; } - if (withinProportion >= autoScalerConfig.getTriggerScaleInThresholdFrequency()) { + if (withinProportion >= defaultAutoScalerConfig.getTriggerScaleInThresholdFrequency()) { // Do Scale in - int taskCount = currentActiveTaskCount - autoScalerConfig.getScaleInStep(); - if (currentActiveTaskCount == autoScalerConfig.getTaskCountMin()) { + int taskCount = currentActiveTaskCount - defaultAutoScalerConfig.getScaleInStep(); + if (currentActiveTaskCount == defaultAutoScalerConfig.getTaskCountMin()) { log.info("CurrentActiveTaskCount reach task count Min limit, skip to scale in tasks for dataSource [%s].", dataSource); return -1; } else { - desireActiveTaskCount = Math.max(taskCount, autoScalerConfig.getTaskCountMin()); + desireActiveTaskCount = Math.max(taskCount, defaultAutoScalerConfig.getTaskCountMin()); } log.debug("Start to scale in tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desireActiveTaskCount, dataSource); return desireActiveTaskCount; @@ -237,8 +237,8 @@ private Integer computeDesireTaskCount(List lags) return -1; } - public AutoScalerConfig getAutoScalerConfig() + public DefaultAutoScalerConfig getAutoScalerConfig() { - return autoScalerConfig; + return defaultAutoScalerConfig; } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScalerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScalerConfig.java new file mode 100644 index 000000000000..afc3bec98f9d --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScalerConfig.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.seekablestream.supervisor.autoscaler; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +public class DefaultAutoScalerConfig implements AutoScalerConfig +{ + private final long metricsCollectionIntervalMillis; + private final long metricsCollectionRangeMillis; + private final long dynamicCheckStartDelayMillis; + private final long dynamicCheckPeriod; + private final long scaleOutThreshold; + private final long scaleInThreshold; + private final double triggerScaleOutThresholdFrequency; + private final double triggerScaleInThresholdFrequency; + private final int taskCountMax; + private final int taskCountMin; + private final int scaleInStep; + private final int scaleOutStep; + private final boolean enableTaskAutoscaler; + private final String autoScalerStrategy; + private final long minTriggerDynamicFrequencyMillis; + + @JsonCreator + public DefaultAutoScalerConfig( + @Nullable @JsonProperty("metricsCollectionIntervalMillis") Long metricsCollectionIntervalMillis, + @Nullable @JsonProperty("metricsCollectionRangeMillis") Long metricsCollectionRangeMillis, + @Nullable @JsonProperty("dynamicCheckStartDelayMillis") Long dynamicCheckStartDelayMillis, + @Nullable @JsonProperty("dynamicCheckPeriod") Long dynamicCheckPeriod, + @Nullable @JsonProperty("scaleOutThreshold") Long scaleOutThreshold, + @Nullable @JsonProperty("scaleInThreshold") Long scaleInThreshold, + @Nullable @JsonProperty("triggerScaleOutThresholdFrequency") Double triggerScaleOutThresholdFrequency, + @Nullable @JsonProperty("triggerScaleInThresholdFrequency") Double triggerScaleInThresholdFrequency, + @Nullable @JsonProperty("taskCountMax") Integer taskCountMax, + @Nullable @JsonProperty("taskCountMin") Integer taskCountMin, + @Nullable @JsonProperty("scaleInStep") Integer scaleInStep, + @Nullable @JsonProperty("scaleOutStep") Integer scaleOutStep, + @Nullable @JsonProperty("enableTaskAutoscaler") Boolean enableTaskAutoscaler, + @Nullable @JsonProperty("autoScalerStrategy") String autoScalerStrategy, + @Nullable @JsonProperty("minTriggerDynamicFrequencyMillis") Long minTriggerDynamicFrequencyMillis) + { + this.metricsCollectionIntervalMillis = metricsCollectionIntervalMillis != null ? metricsCollectionIntervalMillis : 30000; + this.metricsCollectionRangeMillis = metricsCollectionRangeMillis != null ? metricsCollectionRangeMillis : 600000; + this.dynamicCheckStartDelayMillis = dynamicCheckStartDelayMillis != null ? dynamicCheckStartDelayMillis : 300000; + this.dynamicCheckPeriod = dynamicCheckPeriod != null ? dynamicCheckPeriod : 60000; + this.scaleOutThreshold = scaleOutThreshold != null ? scaleOutThreshold : 6000000; + this.scaleInThreshold = scaleInThreshold != null ? scaleInThreshold : 1000000; + this.triggerScaleOutThresholdFrequency = triggerScaleOutThresholdFrequency != null ? triggerScaleOutThresholdFrequency : 0.3; + this.triggerScaleInThresholdFrequency = triggerScaleInThresholdFrequency != null ? triggerScaleInThresholdFrequency : 0.9; + this.taskCountMax = taskCountMax != null ? taskCountMax : 4; + this.taskCountMin = taskCountMin != null ? taskCountMin : 1; + this.scaleInStep = scaleInStep != null ? scaleInStep : 1; + this.scaleOutStep = scaleOutStep != null ? scaleOutStep : 2; + this.enableTaskAutoscaler = enableTaskAutoscaler != null ? enableTaskAutoscaler : false; + this.autoScalerStrategy = autoScalerStrategy != null ? autoScalerStrategy : "default"; + this.minTriggerDynamicFrequencyMillis = minTriggerDynamicFrequencyMillis != null ? minTriggerDynamicFrequencyMillis : 600000; + } + + @JsonProperty + public long getMetricsCollectionIntervalMillis() + { + return metricsCollectionIntervalMillis; + } + + @JsonProperty + public long getMetricsCollectionRangeMillis() + { + return metricsCollectionRangeMillis; + } + + @JsonProperty + public long getDynamicCheckStartDelayMillis() + { + return dynamicCheckStartDelayMillis; + } + + @JsonProperty + public long getDynamicCheckPeriod() + { + return dynamicCheckPeriod; + } + + @JsonProperty + public long getScaleOutThreshold() + { + return scaleOutThreshold; + } + + @JsonProperty + public long getScaleInThreshold() + { + return scaleInThreshold; + } + + @JsonProperty + public double getTriggerScaleOutThresholdFrequency() + { + return triggerScaleOutThresholdFrequency; + } + + @JsonProperty + public double getTriggerScaleInThresholdFrequency() + { + return triggerScaleInThresholdFrequency; + } + + @Override + @JsonProperty + public int getTaskCountMax() + { + return taskCountMax; + } + + @Override + @JsonProperty + public int getTaskCountMin() + { + return taskCountMin; + } + + @JsonProperty + public int getScaleInStep() + { + return scaleInStep; + } + + @JsonProperty + public int getScaleOutStep() + { + return scaleOutStep; + } + + @Override + @JsonProperty + public boolean getEnableTaskAutoscaler() + { + return enableTaskAutoscaler; + } + + @Override + @JsonProperty + public String getAutoScalerStrategy() + { + return autoScalerStrategy; + } + + @Override + @JsonProperty + public long getMinTriggerDynamicFrequencyMillis() + { + return minTriggerDynamicFrequencyMillis; + } +} diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index cfcb013b71fb..0445c9d4afb5 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -46,8 +46,8 @@ import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorReportPayload; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorSpec; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorTuningConfig; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScaler; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScalerConfig; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DummyAutoScaler; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.granularity.Granularities; @@ -531,7 +531,7 @@ public void testAutoScalerCreated() EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, AutoScalerConfig.class)).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, DefaultAutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); @@ -577,7 +577,7 @@ public void testDefaultAutoScalerConfigCreatedWithDefault() EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", "1", "enableTaskAutoscaler", true), AutoScalerConfig.class)).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", "1", "enableTaskAutoscaler", true), DefaultAutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); @@ -600,19 +600,19 @@ public void testDefaultAutoScalerConfigCreatedWithDefault() SupervisorTaskAutoscaler autoscaler = spec.createAutoscaler(supervisor4); Assert.assertTrue(autoscaler instanceof DefaultAutoScaler); DefaultAutoScaler defaultAutoScaler = (DefaultAutoScaler) autoscaler; - AutoScalerConfig autoScalerConfig = defaultAutoScaler.getAutoScalerConfig(); - Assert.assertEquals(autoScalerConfig.getMetricsCollectionIntervalMillis(), 1); - Assert.assertEquals(autoScalerConfig.getMetricsCollectionRangeMillis(), 600000); - Assert.assertEquals(autoScalerConfig.getDynamicCheckStartDelayMillis(), 300000); - Assert.assertEquals(autoScalerConfig.getDynamicCheckPeriod(), 60000); - Assert.assertEquals(autoScalerConfig.getScaleOutThreshold(), 6000000); - Assert.assertEquals(autoScalerConfig.getScaleInThreshold(), 1000000); - Assert.assertEquals(autoScalerConfig.getTaskCountMax(), 4); - Assert.assertEquals(autoScalerConfig.getTaskCountMin(), 1); - Assert.assertEquals(autoScalerConfig.getScaleInStep(), 1); - Assert.assertEquals(autoScalerConfig.getScaleOutStep(), 2); - Assert.assertEquals(autoScalerConfig.getAutoScalerStrategy(), "default"); - Assert.assertEquals(autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), 600000); + DefaultAutoScalerConfig defaultAutoScalerConfig = defaultAutoScaler.getAutoScalerConfig(); + Assert.assertEquals(defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), 1); + Assert.assertEquals(defaultAutoScalerConfig.getMetricsCollectionRangeMillis(), 600000); + Assert.assertEquals(defaultAutoScalerConfig.getDynamicCheckStartDelayMillis(), 300000); + Assert.assertEquals(defaultAutoScalerConfig.getDynamicCheckPeriod(), 60000); + Assert.assertEquals(defaultAutoScalerConfig.getScaleOutThreshold(), 6000000); + Assert.assertEquals(defaultAutoScalerConfig.getScaleInThreshold(), 1000000); + Assert.assertEquals(defaultAutoScalerConfig.getTaskCountMax(), 4); + Assert.assertEquals(defaultAutoScalerConfig.getTaskCountMin(), 1); + Assert.assertEquals(defaultAutoScalerConfig.getScaleInStep(), 1); + Assert.assertEquals(defaultAutoScalerConfig.getScaleOutStep(), 2); + Assert.assertEquals(defaultAutoScalerConfig.getAutoScalerStrategy(), "default"); + Assert.assertEquals(defaultAutoScalerConfig.getMinTriggerDynamicFrequencyMillis(), 600000); } @Test @@ -641,7 +641,7 @@ public void testSeekableStreamSupervisorSpecWithScaleOut() throws InterruptedExc LagStats lagStats = supervisor.computeLagStats(); - DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(), AutoScalerConfig.class), spec); + DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(), DefaultAutoScalerConfig.class), spec); supervisor.start(); autoScaler.start(); supervisor.runInternal(); @@ -679,7 +679,7 @@ public void testSeekableStreamSupervisorSpecWithScaleIn() throws InterruptedExce EasyMock.replay(taskMaster); TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); - DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleInProperties(), AutoScalerConfig.class), spec); + DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleInProperties(), DefaultAutoScalerConfig.class), spec); supervisor.start(); autoScaler.start(); supervisor.runInternal(); @@ -781,7 +781,7 @@ private SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boolean scal false, new Period("PT30M"), null, - null, mapper.convertValue(getScaleOutProperties(), AutoScalerConfig.class), null + null, mapper.convertValue(getScaleOutProperties(), DefaultAutoScalerConfig.class), null ) {}; } else { return new SeekableStreamSupervisorIOConfig( @@ -795,7 +795,7 @@ private SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boolean scal false, new Period("PT30M"), null, - null, mapper.convertValue(getScaleInProperties(), AutoScalerConfig.class), null + null, mapper.convertValue(getScaleInProperties(), DefaultAutoScalerConfig.class), null ) {}; } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index 58dfcae1c54c..91ea8e133e6b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -59,7 +59,7 @@ import org.apache.druid.indexing.seekablestream.common.StreamPartition; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager.SeekableStreamExceptionEvent; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager.SeekableStreamState; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScalerConfig; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; @@ -830,7 +830,7 @@ private static SeekableStreamSupervisorIOConfig getIOConfig() false, new Period("PT30M"), null, - null, OBJECT_MAPPER.convertValue(getProperties(), AutoScalerConfig.class), null + null, OBJECT_MAPPER.convertValue(getProperties(), DefaultAutoScalerConfig.class), null ) { }; From 32fffa955c9ab06ed0a57378e95371e517794f4f Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 4 Feb 2021 13:22:00 +0800 Subject: [PATCH 34/47] modify uts --- .idea/misc.xml | 4 +- .../kafka/supervisor/KafkaSupervisorTest.java | 93 ------------------- .../supervisor/KinesisSupervisorTest.java | 3 +- .../SeekableStreamSupervisorSpecTest.java | 43 +++++++-- .../SeekableStreamSupervisorStateTest.java | 4 +- 5 files changed, 41 insertions(+), 106 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index fda3f144a2b0..1acf50fe7283 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -429,99 +429,6 @@ public KafkaIndexTaskClient build( autoscaler.stop(); } - - @Test - public void testKafkaSupervisorIOConfigInit() - { - final Map consumerProperties = KafkaConsumerConfigs.getConsumerProperties(); - consumerProperties.put("myCustomKey", "myCustomValue"); - consumerProperties.put("bootstrap.servers", kafkaHost); - - //autoscalerConfig = normal - KafkaSupervisorIOConfig kafkaSupervisorIOConfig = new KafkaSupervisorIOConfig( - topic, - INPUT_FORMAT, - 1, - 1, - new Period("PT1H"), - consumerProperties, - OBJECT_MAPPER.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", 1), DefaultAutoScalerConfig.class), - KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, - new Period("P1D"), - new Period("PT30S"), - true, - new Period("PT30M"), - null, - null, - null - ); - - AutoScalerConfig autoScalerConfig = kafkaSupervisorIOConfig.getAutoscalerConfig(); - Assert.assertNotNull(autoScalerConfig); - Assert.assertTrue(autoScalerConfig instanceof DefaultAutoScalerConfig); - DefaultAutoScalerConfig defaultAutoScalerConfig = (DefaultAutoScalerConfig) autoScalerConfig; - Assert.assertEquals(defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), 1); - Assert.assertEquals(defaultAutoScalerConfig.getMetricsCollectionRangeMillis(), 600000); - Assert.assertEquals(defaultAutoScalerConfig.getDynamicCheckStartDelayMillis(), 300000); - Assert.assertEquals(defaultAutoScalerConfig.getDynamicCheckPeriod(), 60000); - Assert.assertEquals(defaultAutoScalerConfig.getScaleOutThreshold(), 6000000); - Assert.assertEquals(defaultAutoScalerConfig.getScaleInThreshold(), 1000000); - Assert.assertEquals(defaultAutoScalerConfig.getTaskCountMax(), 4); - Assert.assertEquals(defaultAutoScalerConfig.getTaskCountMin(), 1); - Assert.assertEquals(defaultAutoScalerConfig.getScaleInStep(), 1); - Assert.assertEquals(defaultAutoScalerConfig.getScaleOutStep(), 2); - Assert.assertFalse(defaultAutoScalerConfig.getEnableTaskAutoscaler()); - Assert.assertEquals(defaultAutoScalerConfig.getAutoScalerStrategy(), "default"); - Assert.assertEquals(defaultAutoScalerConfig.getMinTriggerDynamicFrequencyMillis(), 600000); - - // autoscalerConfig = null ; - KafkaSupervisorIOConfig kafkaSupervisorIOConfig2 = new KafkaSupervisorIOConfig( - topic, - INPUT_FORMAT, - 1, - 1, - new Period("PT1H"), - consumerProperties, - null, - KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, - new Period("P1D"), - new Period("PT30S"), - true, - new Period("PT30M"), - null, - null, - null - ); - - AutoScalerConfig autoScalerConfig2 = kafkaSupervisorIOConfig2.getAutoscalerConfig(); - Assert.assertNull(autoScalerConfig2); - - // autoscalerConfig = empty ; - KafkaSupervisorIOConfig kafkaSupervisorIOConfig3 = new KafkaSupervisorIOConfig( - topic, - INPUT_FORMAT, - 1, - 1, - new Period("PT1H"), - consumerProperties, - OBJECT_MAPPER.convertValue(new HashMap<>(), DefaultAutoScalerConfig.class), - KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, - new Period("P1D"), - new Period("PT30S"), - true, - new Period("PT30M"), - null, - null, - null - ); - - AutoScalerConfig autoScalerConfig3 = kafkaSupervisorIOConfig3.getAutoscalerConfig(); - Assert.assertTrue(autoScalerConfig3 instanceof DefaultAutoScalerConfig); - Assert.assertFalse(autoScalerConfig3.getEnableTaskAutoscaler()); - } - - - @Test public void testCreateBaseTaskContexts() throws JsonProcessingException { diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java index 4127364ea724..0230a7694b23 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java @@ -67,7 +67,6 @@ import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager; import org.apache.druid.indexing.seekablestream.supervisor.TaskReportData; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScalerConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; @@ -403,7 +402,7 @@ public void testKinesisIOConfig() 1000, null, null, - OBJECT_MAPPER.convertValue(new HashMap<>(), DefaultAutoScalerConfig.class), + OBJECT_MAPPER.convertValue(new HashMap<>(), AutoScalerConfig.class), false ); } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 0445c9d4afb5..1f0b3905f028 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -46,6 +46,7 @@ import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorReportPayload; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorSpec; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorTuningConfig; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScaler; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScalerConfig; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DummyAutoScaler; @@ -507,6 +508,25 @@ public String toString() }; } + @Test + public void testAutoScalerConfig() + { + AutoScalerConfig autoScalerConfigEmpty = mapper.convertValue(new HashMap<>(), AutoScalerConfig.class); + Assert.assertTrue(autoScalerConfigEmpty instanceof DefaultAutoScalerConfig); + + AutoScalerConfig autoScalerConfigNull = mapper.convertValue(null, AutoScalerConfig.class); + Assert.assertNull(autoScalerConfigNull); + + AutoScalerConfig autoScalerConfigDefault = mapper.convertValue(ImmutableMap.of("autoScalerStrategy", "default"), AutoScalerConfig.class); + Assert.assertTrue(autoScalerConfigDefault instanceof DefaultAutoScalerConfig); + + AutoScalerConfig autoScalerConfigValue = mapper.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", "1"), AutoScalerConfig.class); + Assert.assertTrue(autoScalerConfigValue instanceof DefaultAutoScalerConfig); + DefaultAutoScalerConfig defaultAutoScalerConfig = (DefaultAutoScalerConfig) autoScalerConfigValue; + Assert.assertEquals(defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), 1); + + } + @Test public void testAutoScalerCreated() { @@ -531,7 +551,7 @@ public void testAutoScalerCreated() EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, DefaultAutoScalerConfig.class)).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, AutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); @@ -554,15 +574,24 @@ public void testAutoScalerCreated() SupervisorTaskAutoscaler autoscaler = spec.createAutoscaler(supervisor4); Assert.assertTrue(autoscaler instanceof DefaultAutoScaler); + EasyMock.reset(seekableStreamSupervisorIOConfig); autoscalerConfig.put("enableTaskAutoscaler", false); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, AutoScalerConfig.class)).anyTimes(); + EasyMock.replay(seekableStreamSupervisorIOConfig); SupervisorTaskAutoscaler autoscaler2 = spec.createAutoscaler(supervisor4); Assert.assertTrue(autoscaler2 instanceof DummyAutoScaler); + EasyMock.reset(seekableStreamSupervisorIOConfig); autoscalerConfig.remove("enableTaskAutoscaler"); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, AutoScalerConfig.class)).anyTimes(); + EasyMock.replay(seekableStreamSupervisorIOConfig); SupervisorTaskAutoscaler autoscaler3 = spec.createAutoscaler(supervisor4); - Assert.assertTrue(autoscaler3 instanceof DefaultAutoScaler); + Assert.assertTrue(autoscaler3 instanceof DummyAutoScaler); + EasyMock.reset(seekableStreamSupervisorIOConfig); autoscalerConfig.clear(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, AutoScalerConfig.class)).anyTimes(); + EasyMock.replay(seekableStreamSupervisorIOConfig); Assert.assertTrue(autoscalerConfig.isEmpty()); SupervisorTaskAutoscaler autoscaler4 = spec.createAutoscaler(supervisor4); Assert.assertTrue(autoscaler4 instanceof DummyAutoScaler); @@ -577,7 +606,7 @@ public void testDefaultAutoScalerConfigCreatedWithDefault() EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", "1", "enableTaskAutoscaler", true), DefaultAutoScalerConfig.class)).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", "1", "enableTaskAutoscaler", true), AutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); @@ -641,7 +670,7 @@ public void testSeekableStreamSupervisorSpecWithScaleOut() throws InterruptedExc LagStats lagStats = supervisor.computeLagStats(); - DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(), DefaultAutoScalerConfig.class), spec); + DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(), AutoScalerConfig.class), spec); supervisor.start(); autoScaler.start(); supervisor.runInternal(); @@ -679,7 +708,7 @@ public void testSeekableStreamSupervisorSpecWithScaleIn() throws InterruptedExce EasyMock.replay(taskMaster); TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); - DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleInProperties(), DefaultAutoScalerConfig.class), spec); + DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleInProperties(), AutoScalerConfig.class), spec); supervisor.start(); autoScaler.start(); supervisor.runInternal(); @@ -781,7 +810,7 @@ private SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boolean scal false, new Period("PT30M"), null, - null, mapper.convertValue(getScaleOutProperties(), DefaultAutoScalerConfig.class), null + null, mapper.convertValue(getScaleOutProperties(), AutoScalerConfig.class), null ) {}; } else { return new SeekableStreamSupervisorIOConfig( @@ -795,7 +824,7 @@ private SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boolean scal false, new Period("PT30M"), null, - null, mapper.convertValue(getScaleInProperties(), DefaultAutoScalerConfig.class), null + null, mapper.convertValue(getScaleInProperties(), AutoScalerConfig.class), null ) {}; } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index 91ea8e133e6b..58dfcae1c54c 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -59,7 +59,7 @@ import org.apache.druid.indexing.seekablestream.common.StreamPartition; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager.SeekableStreamExceptionEvent; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager.SeekableStreamState; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScalerConfig; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; @@ -830,7 +830,7 @@ private static SeekableStreamSupervisorIOConfig getIOConfig() false, new Period("PT30M"), null, - null, OBJECT_MAPPER.convertValue(getProperties(), DefaultAutoScalerConfig.class), null + null, OBJECT_MAPPER.convertValue(getProperties(), AutoScalerConfig.class), null ) { }; From 34c2785dd4973dc1fd34b6739464e228654f4d9f Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 4 Feb 2021 14:18:57 +0800 Subject: [PATCH 35/47] modify docs --- .idea/misc.xml | 4 ++-- docs/development/extensions-core/kafka-ingestion.md | 4 ++-- .../kinesis/supervisor/KinesisSupervisorIOConfig.java | 2 +- .../indexing/kinesis/supervisor/KinesisSupervisorTest.java | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index bf2061d7392d..fe39ed623c9c 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - + \ No newline at end of file diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 619df3974fed..ee1cf1ce06c4 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -146,13 +146,13 @@ A sample supervisor spec is shown below: |`lateMessageRejectionStartDateTime`|ISO8601 DateTime|Configure tasks to reject messages with timestamps earlier than this date time; for example if this is set to `2016-01-01T11:00Z` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline).|no (default == none)| |`lateMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps earlier than this period before the task was created; for example if this is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline). Please note that only one of `lateMessageRejectionPeriod` or `lateMessageRejectionStartDateTime` can be specified.|no (default == none)| |`earlyMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps later than this period after the task reached its taskDuration; for example if this is set to `PT1H`, the taskDuration is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps later than *2016-01-01T14:00Z* will be dropped. **Note:** Tasks sometimes run past their task duration, for example, in cases of supervisor failover. Setting earlyMessageRejectionPeriod too low may cause messages to be dropped unexpectedly whenever a task runs past its originally configured task duration.|no (default == none)| -|`autoscalerConfigDefault`|Object|`autoscalerConfigDefault` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for Kafka indexing as of now. See [Tasks Autoscaler Properties](#Tasks Autoscaler Properties) for details.|no (default == null)| +|`autoscalerConfig`|Object|`autoscalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for Kafka indexing as of now. See [Tasks Autoscaler Properties](#Tasks Autoscaler Properties) for details.|no (default == null)| #### Tasks Autoscaler Properties | Property | Description | Default | | ------------- | ------------- | ------------- | -| `enableTaskAutoscaler` | whether enable this feature or not. Set false here will disable `autoscaler` even though `autoscalerConfigDefault` is not null| true | +| `enableTaskAutoscaler` | whether enable this feature or not. Set false or ignored here will disable `autoscaler` even though `autoscalerConfig` is not null| false | | `metricsCollectionIntervalMillis` | Define the frequency of lag points collection. | 30000 | | `metricsCollectionRangeMillis` | The total time window of lag collection, Use with `metricsCollectionIntervalMillis`，it means that in the recent `metricsCollectionRangeMill`, collect lag metric points every `metricsCollectionIntervalMillis`. | 600000 | | `scaleOutThreshold` | The Threshold of scale out action | 6000000 | diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java index 7db2cce72271..dac32e519777 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java @@ -94,7 +94,7 @@ public KinesisSupervisorIOConfig( // for now dynamic Allocation Tasks is not supported here // throw UnsupportedOperationException in case someone sets this on a kinesis supervisor spec. if (autoscalerConfig != null) { - throw new UnsupportedOperationException("Tasks auto scaler for kinesis is not supported yet."); + throw new UnsupportedOperationException("Tasks auto scaler for kinesis is not supported yet. Please remove autoscalerConfig or set it null!"); } this.endpoint = endpoint != null diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java index 0230a7694b23..ec1c07c7c6ee 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java @@ -374,8 +374,8 @@ public void testKinesisIOConfig() null, false ); - AutoScalerConfig autoscalerConfigDefault = kinesisSupervisorIOConfig.getAutoscalerConfig(); - Assert.assertNull(autoscalerConfigDefault); + AutoScalerConfig autoscalerConfig = kinesisSupervisorIOConfig.getAutoscalerConfig(); + Assert.assertNull(autoscalerConfig); } catch (Exception ex) { e = ex; From eb95830fe960ab0d28d22103b1d115a874fa04c7 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 4 Feb 2021 18:56:42 +0800 Subject: [PATCH 36/47] fix checkstyle --- .../druid/indexing/kafka/supervisor/KafkaSupervisorTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 1acf50fe7283..16edcf2c2396 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -70,7 +70,6 @@ import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorSpec; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager; import org.apache.druid.indexing.seekablestream.supervisor.TaskReportData; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScalerConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; From 7de0f10e7b7a6f802d66983fed928dcf19233049 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 4 Feb 2021 18:57:48 +0800 Subject: [PATCH 37/47] revert misc.xml --- .idea/misc.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.idea/misc.xml b/.idea/misc.xml index fe39ed623c9c..bf2061d7392d 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -84,7 +84,7 @@ - + - \ No newline at end of file + From ce5945b18155d058f6899ea68db26147c3e03015 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 5 Feb 2021 10:56:22 +0800 Subject: [PATCH 38/47] modify uts --- .../overlord/supervisor/SupervisorManagerTest.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java index 82e98167ffb6..6b93f84d1625 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManagerTest.java @@ -22,7 +22,6 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; import org.apache.druid.indexing.overlord.DataSourceMetadata; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.metadata.MetadataSupervisorManager; import org.easymock.Capture; @@ -405,12 +404,6 @@ public Supervisor createSupervisor() return supervisor; } - @Override - public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) - { - return null; - } - @Override public boolean isSuspended() { From 85660b7614a43de30b1859f0e275213b7e1e2343 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 5 Feb 2021 23:23:49 +0800 Subject: [PATCH 39/47] reviewed code change --- .../supervisor/SeekableStreamSupervisor.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index aa1636b3cec0..31a4db5118a9 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -355,7 +355,7 @@ public void handle() } } if (autoScalerConfig != null && nowTime - dynamicTriggerLastRunTime < autoScalerConfig.getMinTriggerDynamicFrequencyMillis()) { - log.info("NowTime - dynamicTriggerLastRunTime is [%s]. Defined minTriggerDynamicFrequency is [%s] for dataSource [%s], CLAM DOWN NOW !", nowTime - dynamicTriggerLastRunTime, autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), dataSource); + log.info("NowTime - dynamicTriggerLastRunTime is [%s]. Defined minTriggerDynamicFrequency is [%s] for dataSource [%s], CALM DOWN NOW !", nowTime - dynamicTriggerLastRunTime, autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), dataSource); return; } @@ -367,7 +367,7 @@ public void handle() } } catch (Exception ex) { - log.warn(ex, "Error, when parse DynamicAllocationTasksNotice"); + log.warn(ex, "Error parsing DynamicAllocationTasksNotice"); } } } @@ -378,36 +378,36 @@ public void handle() * First of all, call gracefulShutdownInternal() which will change the state of current datasource ingest tasks from reading to publishing. * Secondly, clear all the stateful data structures: activelyReadingTaskGroups, partitionGroups, partitionOffsets, pendingCompletionTaskGroups, partitionIds. These structures will be rebuiled next 'RunNotice'. * Finally, change taskCount in SeekableStreamSupervisorIOConfig and sync it to MetaStorage. - * After changed taskCount in SeekableStreamSupervisorIOConfig, next RunNotice will ceate scaled number of ingest tasks without resubmitting supervisors. - * @param desireActiveTaskCount desire taskCount compute from autoscaler - * @return Boolean flag, do scale action successfully or not. If true , it will take at least 'minTriggerDynamicFrequency' before next 'dynamicAllocatie'. + * After changed taskCount in SeekableStreamSupervisorIOConfig, next RunNotice will create scaled number of ingest tasks without resubmitting supervisors. + * @param desiredActiveTaskCount desired taskCount compute from autoscaler + * @return Boolean flag, do scale action successfully or not. If true , it will take at least 'minTriggerDynamicFrequency' before next 'dynamicAllocate'. * If false, it will do 'dynamicAllocate' again after 'dynamicCheckPeriod'. * @throws InterruptedException * @throws ExecutionException * @throws TimeoutException */ - private boolean dynamicAllocate(Integer desireActiveTaskCount) throws InterruptedException, ExecutionException, TimeoutException + private boolean dynamicAllocate(Integer desiredActiveTaskCount) throws InterruptedException, ExecutionException, TimeoutException { int currentActiveTaskCount; Collection activeTaskGroups = activelyReadingTaskGroups.values(); currentActiveTaskCount = activeTaskGroups.size(); - if (desireActiveTaskCount == -1) { + if (desiredActiveTaskCount == -1 || desiredActiveTaskCount == currentActiveTaskCount) { return false; } else { - log.debug("Start to scale action tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desireActiveTaskCount, dataSource); + log.debug("Start to scale action tasks, current active task number [%s] and desired task number is [%s] for dataSource [%s].", currentActiveTaskCount, desiredActiveTaskCount, dataSource); gracefulShutdownInternal(); + changeTaskCountInIOConfig(desiredActiveTaskCount); // clear everything clearAllocationInfos(); - log.info("Change taskCount to [%s] for dataSource [%s].", desireActiveTaskCount, dataSource); - changeTaskCountInIOConfig(desireActiveTaskCount); + log.info("Changed taskCount to [%s] for dataSource [%s].", desiredActiveTaskCount, dataSource); return true; } } - private void changeTaskCountInIOConfig(int desireActiveTaskCount) + private void changeTaskCountInIOConfig(int desiredActiveTaskCount) { - ioConfig.setTaskCount(desireActiveTaskCount); + ioConfig.setTaskCount(desiredActiveTaskCount); try { Optional supervisorManager = taskMaster.getSupervisorManager(); if (supervisorManager.isPresent()) { From feb3e1e88f68aed4f3ce8a3459d14caf85b663d6 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 5 Feb 2021 23:30:09 +0800 Subject: [PATCH 40/47] reviewed code change --- .../supervisor/autoscaler/DefaultAutoScaler.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java index 82bf3b1d4854..031825a37183 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java @@ -206,7 +206,7 @@ private Integer computeDesireTaskCount(List lags) log.info("CurrentActiveTaskCount is lower than 0 ??? skip [%s].", dataSource); return -1; } - int desireActiveTaskCount; + int desiredActiveTaskCount; if (beyondProportion >= defaultAutoScalerConfig.getTriggerScaleOutThresholdFrequency()) { // Do Scale out @@ -215,10 +215,10 @@ private Integer computeDesireTaskCount(List lags) log.info("CurrentActiveTaskCount reach task count Max limit, skip to scale out tasks for dataSource [%s].", dataSource); return -1; } else { - desireActiveTaskCount = Math.min(taskCount, defaultAutoScalerConfig.getTaskCountMax()); + desiredActiveTaskCount = Math.min(taskCount, defaultAutoScalerConfig.getTaskCountMax()); } - return desireActiveTaskCount; + return desiredActiveTaskCount; } if (withinProportion >= defaultAutoScalerConfig.getTriggerScaleInThresholdFrequency()) { @@ -228,10 +228,10 @@ private Integer computeDesireTaskCount(List lags) log.info("CurrentActiveTaskCount reach task count Min limit, skip to scale in tasks for dataSource [%s].", dataSource); return -1; } else { - desireActiveTaskCount = Math.max(taskCount, defaultAutoScalerConfig.getTaskCountMin()); + desiredActiveTaskCount = Math.max(taskCount, defaultAutoScalerConfig.getTaskCountMin()); } - log.debug("Start to scale in tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desireActiveTaskCount, dataSource); - return desireActiveTaskCount; + log.debug("Start to scale in tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desiredActiveTaskCount, dataSource); + return desiredActiveTaskCount; } return -1; From b6632d6c713f2bf87905de0ebb83756d3716edeb Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 23 Feb 2021 14:07:26 +0800 Subject: [PATCH 41/47] code reviewed --- .../extensions-core/kafka-ingestion.md | 35 ++++---- .../supervisor/SeekableStreamSupervisor.java | 84 +++++++++++-------- .../SeekableStreamSupervisorIOConfig.java | 11 ++- .../autoscaler/DefaultAutoScalerConfig.java | 25 ++++-- .../SeekableStreamSupervisorSpecTest.java | 27 +++++- pom.xml | 5 ++ website/i18n/en.json | 9 ++ 7 files changed, 131 insertions(+), 65 deletions(-) diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index ee1cf1ce06c4..16c457da3db1 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -146,26 +146,25 @@ A sample supervisor spec is shown below: |`lateMessageRejectionStartDateTime`|ISO8601 DateTime|Configure tasks to reject messages with timestamps earlier than this date time; for example if this is set to `2016-01-01T11:00Z` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline).|no (default == none)| |`lateMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps earlier than this period before the task was created; for example if this is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline). Please note that only one of `lateMessageRejectionPeriod` or `lateMessageRejectionStartDateTime` can be specified.|no (default == none)| |`earlyMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps later than this period after the task reached its taskDuration; for example if this is set to `PT1H`, the taskDuration is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps later than *2016-01-01T14:00Z* will be dropped. **Note:** Tasks sometimes run past their task duration, for example, in cases of supervisor failover. Setting earlyMessageRejectionPeriod too low may cause messages to be dropped unexpectedly whenever a task runs past its originally configured task duration.|no (default == none)| -|`autoscalerConfig`|Object|`autoscalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for Kafka indexing as of now. See [Tasks Autoscaler Properties](#Tasks Autoscaler Properties) for details.|no (default == null)| +|`autoscalerConfig`|Object|`autoscalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for Kafka indexing as of now. See [Tasks Autoscaler Properties](#task-autoscaler-properties) for details.|no (default == null)| -#### Tasks Autoscaler Properties - -| Property | Description | Default | +### Task Autoscaler Properties +| Property | Description | Required | | ------------- | ------------- | ------------- | -| `enableTaskAutoscaler` | whether enable this feature or not. Set false or ignored here will disable `autoscaler` even though `autoscalerConfig` is not null| false | -| `metricsCollectionIntervalMillis` | Define the frequency of lag points collection. | 30000 | -| `metricsCollectionRangeMillis` | The total time window of lag collection, Use with `metricsCollectionIntervalMillis`，it means that in the recent `metricsCollectionRangeMill`, collect lag metric points every `metricsCollectionIntervalMillis`. | 600000 | -| `scaleOutThreshold` | The Threshold of scale out action | 6000000 | -| `triggerScaleOutThresholdFrequency` | If `triggerScaleOutThresholdFrequency` percent of lag points are higher than `scaleOutThreshold`, then do scale out action. | 0.3 | -| `scaleInThreshold` | The Threshold of scale in action | 1000000 | -| `triggerScaleInThresholdFrequency` | If `triggerScaleInThresholdFrequency` percent of lag points are lower than `scaleOutThreshold`, then do scale in action. | 0.9 | -| `dynamicCheckStartDelayMillis` | Number of milliseconds after supervisor starts when first check scale logic. | 300000 | -| `dynamicCheckPeriod` | the frequency of checking whether to do scale action | 60000 | -| `taskCountMax` | Maximum value of task count | 4 | -| `taskCountMin` | Minimum value of task count | 1 | -| `scaleInStep` | How many tasks to reduce at a time | 1 | -| `scaleOutStep` | How many tasks to add at a time | 2 | -| `minTriggerDynamicFrequencyMillis` | Minimum time interval between two scale actions | 600000 | +| `enableTaskAutoscaler` | Whether enable this feature or not. Not setting or setting to false will disable `autoscaler` even though `autoscalerConfig` is not null| no (default == false) | +| `metricsCollectionIntervalMillis` | Define the frequency of lag points collection. | no (default == 30000) | +| `metricsCollectionRangeMillis` | The total time window of lag collection, Use with `metricsCollectionIntervalMillis`，it means that in the recent `metricsCollectionRangeMillis`, collect lag metric points every `metricsCollectionIntervalMillis`. | no (default == 600000) | +| `scaleOutThreshold` | The Threshold of scale out action | no (default == 6000000) | +| `triggerScaleOutThresholdFrequency` | If `triggerScaleOutThresholdFrequency` percent of lag points are higher than `scaleOutThreshold`, then do scale out action. | no (default == 0.3) | +| `scaleInThreshold` | The Threshold of scale in action | no (default == 1000000) | +| `triggerScaleInThresholdFrequency` | If `triggerScaleInThresholdFrequency` percent of lag points are lower than `scaleOutThreshold`, then do scale in action. | no (default == 0.9) | +| `dynamicCheckStartDelayMillis` | Number of milliseconds after supervisor starts when first check scale logic. | no (default == 300000) | +| `dynamicCheckPeriod` | The frequency of checking whether to do scale action in millis | no (default == 60000) | +| `taskCountMax` | Maximum value of task count. Make Sure `taskCountMax >= taskCountMin` | yes | +| `taskCountMin` | Minimum value of task count. When enable autoscaler, the value of taskCount in `IOConfig` will be ignored, and `taskCountMin` will be the number of tasks that ingestion starts going up to `taskCountMax`| yes | +| `scaleInStep` | How many tasks to reduce at a time | no (default == 1) | +| `scaleOutStep` | How many tasks to add at a time | no (default == 2) | +| `minTriggerDynamicFrequencyMillis` | Minimum time interval between two scale actions | no (default == 600000) | #### More on consumerProperties diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 31a4db5118a9..ea223e78e026 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -340,48 +340,59 @@ private class DynamicAllocationTasksNotice implements Notice @Override public void handle() { - try { - long nowTime = System.currentTimeMillis(); - // Only queue is full and over minTriggerDynamicFrequency can trigger scale out/in - if (spec.isSuspended()) { - log.info("[%s] supervisor is suspended, skip to check dynamic allocate task logic", dataSource); - return; - } - log.debug("PendingCompletionTaskGroups is [%s] for dataSource [%s].", pendingCompletionTaskGroups, dataSource); - for (CopyOnWriteArrayList list : pendingCompletionTaskGroups.values()) { - if (!list.isEmpty()) { - log.info("Still hand off tasks unfinished, skip to do scale action [%s] for dataSource [%s].", pendingCompletionTaskGroups, dataSource); + if (autoScalerConfig == null) { + log.warn("autoScalerConfig is null but dynamic allocation notice is submitted, how can it be ?"); + } else { + try { + long nowTime = System.currentTimeMillis(); + if (spec.isSuspended()) { + log.info("Skipping DynamicAllocationTasksNotice execution because [%s] supervisor is suspended", + dataSource + ); return; } + log.debug("PendingCompletionTaskGroups is [%s] for dataSource [%s]", pendingCompletionTaskGroups, + dataSource + ); + for (CopyOnWriteArrayList list : pendingCompletionTaskGroups.values()) { + if (!list.isEmpty()) { + log.info( + "Skipping DynamicAllocationTasksNotice execution for datasource [%s] because following tasks are pending [%s]", + dataSource, pendingCompletionTaskGroups + ); + return; + } + } + if (nowTime - dynamicTriggerLastRunTime < autoScalerConfig.getMinTriggerDynamicFrequencyMillis()) { + log.info( + "DynamicAllocationTasksNotice submitted again in [%d] millis, minTriggerDynamicFrequency is [%s] for dataSource [%s], skipping it!", + nowTime - dynamicTriggerLastRunTime, autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), dataSource + ); + return; + } + Integer desriedTaskCount = scaleAction.call(); + boolean allocationSuccess = dynamicAllocate(desriedTaskCount); + if (allocationSuccess) { + dynamicTriggerLastRunTime = nowTime; + } } - if (autoScalerConfig != null && nowTime - dynamicTriggerLastRunTime < autoScalerConfig.getMinTriggerDynamicFrequencyMillis()) { - log.info("NowTime - dynamicTriggerLastRunTime is [%s]. Defined minTriggerDynamicFrequency is [%s] for dataSource [%s], CALM DOWN NOW !", nowTime - dynamicTriggerLastRunTime, autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), dataSource); - return; - } - - Integer desriedTaskCount = scaleAction.call(); - boolean allocationSuccess = dynamicAllocate(desriedTaskCount); - - if (allocationSuccess) { - dynamicTriggerLastRunTime = nowTime; + catch (Exception ex) { + log.warn(ex, "Error parsing DynamicAllocationTasksNotice"); } } - catch (Exception ex) { - log.warn(ex, "Error parsing DynamicAllocationTasksNotice"); - } } } /** * This method determines how to do scale actions based on collected lag points. * If scale action is triggered : - * First of all, call gracefulShutdownInternal() which will change the state of current datasource ingest tasks from reading to publishing. - * Secondly, clear all the stateful data structures: activelyReadingTaskGroups, partitionGroups, partitionOffsets, pendingCompletionTaskGroups, partitionIds. These structures will be rebuiled next 'RunNotice'. - * Finally, change taskCount in SeekableStreamSupervisorIOConfig and sync it to MetaStorage. - * After changed taskCount in SeekableStreamSupervisorIOConfig, next RunNotice will create scaled number of ingest tasks without resubmitting supervisors. + * First of all, call gracefulShutdownInternal() which will change the state of current datasource ingest tasks from reading to publishing. + * Secondly, clear all the stateful data structures: activelyReadingTaskGroups, partitionGroups, partitionOffsets, pendingCompletionTaskGroups, partitionIds. These structures will be rebuild in the next 'RunNotice'. + * Finally, change the taskCount in SeekableStreamSupervisorIOConfig and sync it to MetadataStorage. + * After the taskCount is changed in SeekableStreamSupervisorIOConfig, next RunNotice will create scaled number of ingest tasks without resubmitting the supervisor. * @param desiredActiveTaskCount desired taskCount compute from autoscaler - * @return Boolean flag, do scale action successfully or not. If true , it will take at least 'minTriggerDynamicFrequency' before next 'dynamicAllocate'. - * If false, it will do 'dynamicAllocate' again after 'dynamicCheckPeriod'. + * @return Boolean flag indicating if scale action was executed or not. If true, it will wait at least 'minTriggerDynamicFrequency' before next 'dynamicAllocate'. + * If false, it will do 'dynamicAllocate' again after 'dynamicCheckPeriod' millis. * @throws InterruptedException * @throws ExecutionException * @throws TimeoutException @@ -395,7 +406,10 @@ private boolean dynamicAllocate(Integer desiredActiveTaskCount) throws Interrupt if (desiredActiveTaskCount == -1 || desiredActiveTaskCount == currentActiveTaskCount) { return false; } else { - log.debug("Start to scale action tasks, current active task number [%s] and desired task number is [%s] for dataSource [%s].", currentActiveTaskCount, desiredActiveTaskCount, dataSource); + log.debug( + "Starting scale action, current active task count is [%d] and desired task count is [%d] for dataSource [%s].", + currentActiveTaskCount, desiredActiveTaskCount, dataSource + ); gracefulShutdownInternal(); changeTaskCountInIOConfig(desiredActiveTaskCount); // clear everything @@ -414,7 +428,7 @@ private void changeTaskCountInIOConfig(int desiredActiveTaskCount) MetadataSupervisorManager metadataSupervisorManager = supervisorManager.get().getMetadataSupervisorManager(); metadataSupervisorManager.insert(dataSource, spec); } else { - log.warn("supervisorManager is null in taskMaster, skip to do scale action for dataSource [%s].", dataSource); + log.warn("supervisorManager is null in taskMaster, skipping scale action for dataSource [%s].", dataSource); } } catch (Exception e) { @@ -651,7 +665,7 @@ public SeekableStreamSupervisor( int workerThreads; int chatThreads; if (autoScalerConfig != null && autoScalerConfig.getEnableTaskAutoscaler()) { - log.info("enableTaskAutoscaler for datasource [%s]", dataSource); + log.info("Running Task autoscaler for datasource [%s]", dataSource); workerThreads = (this.tuningConfig.getWorkerThreads() != null ? this.tuningConfig.getWorkerThreads() @@ -661,8 +675,6 @@ public SeekableStreamSupervisor( ? this.tuningConfig.getChatThreads() : Math.min(10, autoScalerConfig.getTaskCountMax() * this.ioConfig.getReplicas())); } else { - log.info("Disable dynamic allocate tasks for [%s]", dataSource); - workerThreads = (this.tuningConfig.getWorkerThreads() != null ? this.tuningConfig.getWorkerThreads() : Math.min(10, this.ioConfig.getTaskCount())); @@ -3684,7 +3696,7 @@ protected void emitLag() /** - * This method compute maxLag, totalLag and avgLag then fill in 'lags' + * This method computes maxLag, totalLag and avgLag * @param partitionLags lags per partition */ protected LagStats computeLags(Map partitionLags) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java index ca25a16fe4ce..df0ee848d21f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java @@ -68,7 +68,14 @@ public SeekableStreamSupervisorIOConfig( this.stream = Preconditions.checkNotNull(stream, "stream cannot be null"); this.inputFormat = inputFormat; this.replicas = replicas != null ? replicas : 1; - this.taskCount = taskCount != null ? taskCount : 1; + // Could be null + this.autoscalerConfig = autoscalerConfig; + // if autoscaler is enable then taskcount will be ignored here. and init taskcount will be equal to taskCountMin + if (autoscalerConfig != null && autoscalerConfig.getEnableTaskAutoscaler()) { + this.taskCount = autoscalerConfig.getTaskCountMin(); + } else { + this.taskCount = taskCount != null ? taskCount : 1; + } this.taskDuration = defaultDuration(taskDuration, "PT1H"); this.startDelay = defaultDuration(startDelay, "PT5S"); this.period = defaultDuration(period, "PT30S"); @@ -90,8 +97,6 @@ public SeekableStreamSupervisorIOConfig( + "both properties lateMessageRejectionStartDateTime " + "and lateMessageRejectionPeriod."); } - // Could be null - this.autoscalerConfig = autoscalerConfig; } private static Duration defaultDuration(final Period period, final String theDefault) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScalerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScalerConfig.java index afc3bec98f9d..f4169529a49e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScalerConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScalerConfig.java @@ -34,8 +34,8 @@ public class DefaultAutoScalerConfig implements AutoScalerConfig private final long scaleInThreshold; private final double triggerScaleOutThresholdFrequency; private final double triggerScaleInThresholdFrequency; - private final int taskCountMax; - private final int taskCountMin; + private int taskCountMax; + private int taskCountMin; private final int scaleInStep; private final int scaleOutStep; private final boolean enableTaskAutoscaler; @@ -52,14 +52,15 @@ public DefaultAutoScalerConfig( @Nullable @JsonProperty("scaleInThreshold") Long scaleInThreshold, @Nullable @JsonProperty("triggerScaleOutThresholdFrequency") Double triggerScaleOutThresholdFrequency, @Nullable @JsonProperty("triggerScaleInThresholdFrequency") Double triggerScaleInThresholdFrequency, - @Nullable @JsonProperty("taskCountMax") Integer taskCountMax, - @Nullable @JsonProperty("taskCountMin") Integer taskCountMin, + @JsonProperty("taskCountMax") Integer taskCountMax, + @JsonProperty("taskCountMin") Integer taskCountMin, @Nullable @JsonProperty("scaleInStep") Integer scaleInStep, @Nullable @JsonProperty("scaleOutStep") Integer scaleOutStep, @Nullable @JsonProperty("enableTaskAutoscaler") Boolean enableTaskAutoscaler, @Nullable @JsonProperty("autoScalerStrategy") String autoScalerStrategy, @Nullable @JsonProperty("minTriggerDynamicFrequencyMillis") Long minTriggerDynamicFrequencyMillis) { + this.enableTaskAutoscaler = enableTaskAutoscaler != null ? enableTaskAutoscaler : false; this.metricsCollectionIntervalMillis = metricsCollectionIntervalMillis != null ? metricsCollectionIntervalMillis : 30000; this.metricsCollectionRangeMillis = metricsCollectionRangeMillis != null ? metricsCollectionRangeMillis : 600000; this.dynamicCheckStartDelayMillis = dynamicCheckStartDelayMillis != null ? dynamicCheckStartDelayMillis : 300000; @@ -68,11 +69,21 @@ public DefaultAutoScalerConfig( this.scaleInThreshold = scaleInThreshold != null ? scaleInThreshold : 1000000; this.triggerScaleOutThresholdFrequency = triggerScaleOutThresholdFrequency != null ? triggerScaleOutThresholdFrequency : 0.3; this.triggerScaleInThresholdFrequency = triggerScaleInThresholdFrequency != null ? triggerScaleInThresholdFrequency : 0.9; - this.taskCountMax = taskCountMax != null ? taskCountMax : 4; - this.taskCountMin = taskCountMin != null ? taskCountMin : 1; + + // Only do taskCountMax and taskCountMin check when autoscaler is enabled. So that users left autoConfig empty{} will not throw any exception and autoscaler is disabled. + // If autoscaler is disabled, no matter what configs are set, they are not used. + if (this.enableTaskAutoscaler) { + if (taskCountMax == null || taskCountMin == null) { + throw new RuntimeException("taskCountMax or taskCountMin can't be null!"); + } else if (taskCountMax < taskCountMin) { + throw new RuntimeException("taskCountMax can't lower than taskCountMin!"); + } + this.taskCountMax = taskCountMax; + this.taskCountMin = taskCountMin; + } + this.scaleInStep = scaleInStep != null ? scaleInStep : 1; this.scaleOutStep = scaleOutStep != null ? scaleOutStep : 2; - this.enableTaskAutoscaler = enableTaskAutoscaler != null ? enableTaskAutoscaler : false; this.autoScalerStrategy = autoScalerStrategy != null ? autoScalerStrategy : "default"; this.minTriggerDynamicFrequencyMillis = minTriggerDynamicFrequencyMillis != null ? minTriggerDynamicFrequencyMillis : 600000; } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 1f0b3905f028..4190454722fa 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -513,6 +513,7 @@ public void testAutoScalerConfig() { AutoScalerConfig autoScalerConfigEmpty = mapper.convertValue(new HashMap<>(), AutoScalerConfig.class); Assert.assertTrue(autoScalerConfigEmpty instanceof DefaultAutoScalerConfig); + Assert.assertFalse(autoScalerConfigEmpty.getEnableTaskAutoscaler()); AutoScalerConfig autoScalerConfigNull = mapper.convertValue(null, AutoScalerConfig.class); Assert.assertNull(autoScalerConfigNull); @@ -525,6 +526,26 @@ public void testAutoScalerConfig() DefaultAutoScalerConfig defaultAutoScalerConfig = (DefaultAutoScalerConfig) autoScalerConfigValue; Assert.assertEquals(defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), 1); + Exception e = null; + try { + AutoScalerConfig autoScalerError = mapper.convertValue(ImmutableMap.of("enableTaskAutoscaler", "true", "taskCountMax", "1", "taskCountMin", "4"), AutoScalerConfig.class); + } + catch (RuntimeException ex) { + e = ex; + } + Assert.assertNotNull(e); + + e = null; + try { + // taskCountMax and taskCountMin couldn't be ignored. + AutoScalerConfig autoScalerError2 = mapper.convertValue(ImmutableMap.of("enableTaskAutoscaler", "true"), AutoScalerConfig.class); + } + catch (RuntimeException ex) { + e = ex; + } + Assert.assertNotNull(e); + + } @Test @@ -606,7 +627,7 @@ public void testDefaultAutoScalerConfigCreatedWithDefault() EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", "1", "enableTaskAutoscaler", true), AutoScalerConfig.class)).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", "1", "enableTaskAutoscaler", true, "taskCountMax", "4", "taskCountMin", "1"), AutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); @@ -709,6 +730,10 @@ public void testSeekableStreamSupervisorSpecWithScaleIn() throws InterruptedExce TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleInProperties(), AutoScalerConfig.class), spec); + + // enable autoscaler so that taskcount config will be ignored and init value of taskCount will use taskCountMin. + Assert.assertEquals(1, (int) supervisor.getIoConfig().getTaskCount()); + supervisor.getIoConfig().setTaskCount(2); supervisor.start(); autoScaler.start(); supervisor.runInternal(); diff --git a/pom.xml b/pom.xml index 97e5c39b5ea2..77c3a0d54821 100644 --- a/pom.xml +++ b/pom.xml @@ -957,6 +957,11 @@ jna 4.5.1 + + org.apache.commons + commons-collections4 + 4.2 + io.dropwizard.metrics metrics-core diff --git a/website/i18n/en.json b/website/i18n/en.json index 9de85a69ad68..f67380992222 100644 --- a/website/i18n/en.json +++ b/website/i18n/en.json @@ -179,6 +179,9 @@ "development/extensions-core/datasketches-tuple": { "title": "DataSketches Tuple Sketch module" }, + "development/extensions-core/druid-aws-rds": { + "title": "Druid AWS RDS Module" + }, "development/extensions-core/druid-basic-security": { "title": "Basic Security" }, @@ -214,6 +217,9 @@ "title": "Amazon Kinesis ingestion", "sidebar_label": "Amazon Kinesis" }, + "development/extensions-core/druid-kubernetes": { + "title": "Kubernetes" + }, "development/extensions-core/lookups-cached-global": { "title": "Globally Cached Lookups" }, @@ -324,6 +330,9 @@ "operations/dump-segment": { "title": "dump-segment tool" }, + "operations/dynamic-config-provider": { + "title": "Dynamic Config Providers" + }, "operations/export-metadata": { "title": "Export Metadata Tool" }, From 00758e647c7137166b3e149607305a54db284dfb Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 25 Feb 2021 17:10:26 +0800 Subject: [PATCH 42/47] code review --- .../extensions-core/kafka-ingestion.md | 15 +- .../MaterializedViewSupervisorSpecTest.java | 4 +- .../supervisor/KafkaSupervisorIOConfig.java | 4 +- .../kafka/supervisor/KafkaSupervisorTest.java | 38 +-- .../supervisor/KinesisSupervisorIOConfig.java | 10 +- .../supervisor/KinesisSupervisorTest.java | 8 +- .../supervisor/SupervisorManager.java | 14 +- .../supervisor/SeekableStreamSupervisor.java | 34 ++- .../SeekableStreamSupervisorIOConfig.java | 12 +- .../SeekableStreamSupervisorSpec.java | 31 +-- .../autoscaler/AutoScalerConfig.java | 13 +- .../autoscaler/DefaultAutoScaler.java | 244 ------------------ .../autoscaler/LagBasedAutoScaler.java | 242 +++++++++++++++++ ...fig.java => LagBasedAutoScalerConfig.java} | 90 +++---- ...utoScaler.java => NoopTaskAutoScaler.java} | 7 +- .../OverlordSecurityResourceFilterTest.java | 8 +- .../supervisor/SupervisorResourceTest.java | 4 +- .../SeekableStreamSupervisorSpecTest.java | 201 +++++++-------- .../SeekableStreamSupervisorStateTest.java | 32 +-- ...ervisor_with_autoscaler_spec_template.json | 14 +- .../overlord/supervisor/Supervisor.java | 4 +- .../overlord/supervisor/SupervisorSpec.java | 4 +- ...ler.java => SupervisorTaskAutoScaler.java} | 2 +- .../indexing/NoopSupervisorSpecTest.java | 4 +- .../SQLMetadataSupervisorManagerTest.java | 4 +- .../druid/metadata/TestSupervisorSpec.java | 4 +- 26 files changed, 513 insertions(+), 534 deletions(-) delete mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java create mode 100644 indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java rename indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/{DefaultAutoScalerConfig.java => LagBasedAutoScalerConfig.java} (60%) rename indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/{DummyAutoScaler.java => NoopTaskAutoScaler.java} (83%) rename server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/{SupervisorTaskAutoscaler.java => SupervisorTaskAutoScaler.java} (95%) diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 673bc99cabc7..810b940b40d8 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -146,25 +146,26 @@ A sample supervisor spec is shown below: |`lateMessageRejectionStartDateTime`|ISO8601 DateTime|Configure tasks to reject messages with timestamps earlier than this date time; for example if this is set to `2016-01-01T11:00Z` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline).|no (default == none)| |`lateMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps earlier than this period before the task was created; for example if this is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline). Please note that only one of `lateMessageRejectionPeriod` or `lateMessageRejectionStartDateTime` can be specified.|no (default == none)| |`earlyMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps later than this period after the task reached its taskDuration; for example if this is set to `PT1H`, the taskDuration is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps later than *2016-01-01T14:00Z* will be dropped. **Note:** Tasks sometimes run past their task duration, for example, in cases of supervisor failover. Setting earlyMessageRejectionPeriod too low may cause messages to be dropped unexpectedly whenever a task runs past its originally configured task duration.|no (default == none)| -|`autoscalerConfig`|Object|`autoscalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for Kafka indexing as of now. See [Tasks Autoscaler Properties](#task-autoscaler-properties) for details.|no (default == null)| +|`autoScalerConfig`|Object|`autoScalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for Kafka indexing as of now. See [Tasks Autoscaler Properties](#Task Autoscaler Properties) for details.|no (default == null)| ### Task Autoscaler Properties | Property | Description | Required | | ------------- | ------------- | ------------- | -| `enableTaskAutoscaler` | Whether enable this feature or not. Not setting or setting to false will disable `autoscaler` even though `autoscalerConfig` is not null| no (default == false) | -| `metricsCollectionIntervalMillis` | Define the frequency of lag points collection. | no (default == 30000) | -| `metricsCollectionRangeMillis` | The total time window of lag collection, Use with `metricsCollectionIntervalMillis`，it means that in the recent `metricsCollectionRangeMillis`, collect lag metric points every `metricsCollectionIntervalMillis`. | no (default == 600000) | +| `enableTaskAutoScaler` | Whether enable this feature or not. Set false or ignored here will disable `autoScaler` even though `autoScalerConfig` is not null| no (default == false) | +| `lagCollectionIntervalMillis` | Define the frequency of lag points collection. | no (default == 30000) | +| `lagCollectionRangeMillis` | The total time window of lag collection, Use with `lagCollectionIntervalMillis`，it means that in the recent `lagCollectionRangeMillis`, collect lag metric points every `lagCollectionIntervalMillis`. | no (default == 600000) | | `scaleOutThreshold` | The Threshold of scale out action | no (default == 6000000) | | `triggerScaleOutThresholdFrequency` | If `triggerScaleOutThresholdFrequency` percent of lag points are higher than `scaleOutThreshold`, then do scale out action. | no (default == 0.3) | | `scaleInThreshold` | The Threshold of scale in action | no (default == 1000000) | | `triggerScaleInThresholdFrequency` | If `triggerScaleInThresholdFrequency` percent of lag points are lower than `scaleOutThreshold`, then do scale in action. | no (default == 0.9) | -| `dynamicCheckStartDelayMillis` | Number of milliseconds after supervisor starts when first check scale logic. | no (default == 300000) | -| `dynamicCheckPeriod` | The frequency of checking whether to do scale action in millis | no (default == 60000) | +| `scaleActionStartDelayMillis` | Number of milliseconds after supervisor starts when first check scale logic. | no (default == 300000) | +| `scaleActionPeriodMillis` | The frequency of checking whether to do scale action in millis | no (default == 60000) | | `taskCountMax` | Maximum value of task count. Make Sure `taskCountMax >= taskCountMin` | yes | | `taskCountMin` | Minimum value of task count. When enable autoscaler, the value of taskCount in `IOConfig` will be ignored, and `taskCountMin` will be the number of tasks that ingestion starts going up to `taskCountMax`| yes | | `scaleInStep` | How many tasks to reduce at a time | no (default == 1) | | `scaleOutStep` | How many tasks to add at a time | no (default == 2) | -| `minTriggerDynamicFrequencyMillis` | Minimum time interval between two scale actions | no (default == 600000) | +| `minTriggerScaleActionFrequencyMillis` | Minimum time interval between two scale actions | no (default == 600000) | +| `autoScalerStrategy` | The algorithm of `autoScaler`. ONLY `lagBased` is supported for now. | no (default == `lagBased`) | #### More on consumerProperties diff --git a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java index da6c5a3d959d..01ebdd0a9310 100644 --- a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java +++ b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java @@ -31,7 +31,7 @@ import org.apache.druid.indexing.overlord.TaskStorage; import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.metadata.MetadataSupervisorManager; import org.apache.druid.metadata.SqlSegmentsMetadataManager; @@ -204,7 +204,7 @@ public void testMaterializedViewSupervisorSpecCreated() Supervisor supervisor = spec.createSupervisor(); Assert.assertTrue(supervisor instanceof MaterializedViewSupervisor); - SupervisorTaskAutoscaler autoscaler = spec.createAutoscaler(supervisor); + SupervisorTaskAutoScaler autoscaler = spec.createAutoscaler(supervisor); Assert.assertNull(autoscaler); try { diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java index c96c0bb2fa2a..6d6a0c3f9750 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java @@ -53,7 +53,7 @@ public KafkaSupervisorIOConfig( @JsonProperty("taskCount") Integer taskCount, @JsonProperty("taskDuration") Period taskDuration, @JsonProperty("consumerProperties") Map consumerProperties, - @Nullable @JsonProperty("autoscalerConfig") AutoScalerConfig autoscalerConfig, + @Nullable @JsonProperty("autoScalerConfig") AutoScalerConfig autoScalerConfig, @JsonProperty("pollTimeout") Long pollTimeout, @JsonProperty("startDelay") Period startDelay, @JsonProperty("period") Period period, @@ -76,7 +76,7 @@ public KafkaSupervisorIOConfig( completionTimeout, lateMessageRejectionPeriod, earlyMessageRejectionPeriod, - autoscalerConfig, + autoScalerConfig, lateMessageRejectionStartDateTime ); diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 16edcf2c2396..547acf510294 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -61,7 +61,7 @@ import org.apache.druid.indexing.overlord.supervisor.SupervisorReport; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; import org.apache.druid.indexing.seekablestream.SeekableStreamEndSequenceNumbers; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.Status; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskTuningConfig; @@ -70,7 +70,7 @@ import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorSpec; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorStateManager; import org.apache.druid.indexing.seekablestream.supervisor.TaskReportData; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScalerConfig; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.LagBasedAutoScalerConfig; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.StringUtils; @@ -261,21 +261,21 @@ public KafkaIndexTaskClient build( } }; - HashMap autoscalerConfig = new HashMap<>(); - autoscalerConfig.put("enableTaskAutoscaler", true); - autoscalerConfig.put("metricsCollectionIntervalMillis", 500); - autoscalerConfig.put("metricsCollectionRangeMillis", 500); - autoscalerConfig.put("scaleOutThreshold", 0); - autoscalerConfig.put("triggerScaleOutThresholdFrequency", 0.0); - autoscalerConfig.put("scaleInThreshold", 1000000); - autoscalerConfig.put("triggerScaleInThresholdFrequency", 0.8); - autoscalerConfig.put("dynamicCheckStartDelayMillis", 0); - autoscalerConfig.put("dynamicCheckPeriod", 100); - autoscalerConfig.put("taskCountMax", 2); - autoscalerConfig.put("taskCountMin", 1); - autoscalerConfig.put("scaleInStep", 1); - autoscalerConfig.put("scaleOutStep", 2); - autoscalerConfig.put("minTriggerDynamicFrequencyMillis", 1200000); + HashMap autoScalerConfig = new HashMap<>(); + autoScalerConfig.put("enableTaskAutoScaler", true); + autoScalerConfig.put("lagCollectionIntervalMillis", 500); + autoScalerConfig.put("lagCollectionRangeMillis", 500); + autoScalerConfig.put("scaleOutThreshold", 0); + autoScalerConfig.put("triggerScaleOutThresholdFrequency", 0.0); + autoScalerConfig.put("scaleInThreshold", 1000000); + autoScalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoScalerConfig.put("scaleActionStartDelayMillis", 0); + autoScalerConfig.put("scaleActionPeriodMillis", 100); + autoScalerConfig.put("taskCountMax", 2); + autoScalerConfig.put("taskCountMin", 1); + autoScalerConfig.put("scaleInStep", 1); + autoScalerConfig.put("scaleOutStep", 2); + autoScalerConfig.put("minTriggerScaleActionFrequencyMillis", 1200000); final Map consumerProperties = KafkaConsumerConfigs.getConsumerProperties(); consumerProperties.put("myCustomKey", "myCustomValue"); @@ -288,7 +288,7 @@ public KafkaIndexTaskClient build( 1, new Period("PT1H"), consumerProperties, - OBJECT_MAPPER.convertValue(autoscalerConfig, DefaultAutoScalerConfig.class), + OBJECT_MAPPER.convertValue(autoScalerConfig, LagBasedAutoScalerConfig.class), KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), @@ -361,7 +361,7 @@ public KafkaIndexTaskClient build( rowIngestionMetersFactory ); - SupervisorTaskAutoscaler autoscaler = testableSupervisorSpec.createAutoscaler(supervisor); + SupervisorTaskAutoScaler autoscaler = testableSupervisorSpec.createAutoscaler(supervisor); final KafkaSupervisorTuningConfig tuningConfig = supervisor.getTuningConfig(); diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java index dac32e519777..b43cece0e522 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorIOConfig.java @@ -30,6 +30,8 @@ import org.joda.time.DateTime; import org.joda.time.Period; +import javax.annotation.Nullable; + public class KinesisSupervisorIOConfig extends SeekableStreamSupervisorIOConfig { private final String endpoint; @@ -71,7 +73,7 @@ public KinesisSupervisorIOConfig( @JsonProperty("fetchDelayMillis") Integer fetchDelayMillis, @JsonProperty("awsAssumedRoleArn") String awsAssumedRoleArn, @JsonProperty("awsExternalId") String awsExternalId, - @JsonProperty("autoscalerConfig") AutoScalerConfig autoscalerConfig, + @Nullable @JsonProperty("autoScalerConfig") AutoScalerConfig autoScalerConfig, @JsonProperty("deaggregate") boolean deaggregate ) { @@ -87,14 +89,14 @@ public KinesisSupervisorIOConfig( completionTimeout, lateMessageRejectionPeriod, earlyMessageRejectionPeriod, - null, + autoScalerConfig, lateMessageRejectionStartDateTime ); // for now dynamic Allocation Tasks is not supported here // throw UnsupportedOperationException in case someone sets this on a kinesis supervisor spec. - if (autoscalerConfig != null) { - throw new UnsupportedOperationException("Tasks auto scaler for kinesis is not supported yet. Please remove autoscalerConfig or set it null!"); + if (autoScalerConfig != null) { + throw new UnsupportedOperationException("Tasks auto scaler for kinesis is not supported yet. Please remove autoScalerConfig or set it to null!"); } this.endpoint = endpoint != null diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java index ec1c07c7c6ee..5d1d0f6a0099 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisorTest.java @@ -374,8 +374,8 @@ public void testKinesisIOConfig() null, false ); - AutoScalerConfig autoscalerConfig = kinesisSupervisorIOConfig.getAutoscalerConfig(); - Assert.assertNull(autoscalerConfig); + AutoScalerConfig autoScalerConfig = kinesisSupervisorIOConfig.getAutoscalerConfig(); + Assert.assertNull(autoScalerConfig); } catch (Exception ex) { e = ex; @@ -411,10 +411,6 @@ public void testKinesisIOConfig() } Assert.assertNotNull(e); Assert.assertTrue(e instanceof UnsupportedOperationException); - - - - } @Test diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java index bf502473e3ba..b638fcfbd8b4 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorManager.java @@ -23,7 +23,7 @@ import com.google.common.base.Preconditions; import com.google.inject.Inject; import org.apache.druid.indexing.overlord.DataSourceMetadata; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.lifecycle.LifecycleStart; import org.apache.druid.java.util.common.lifecycle.LifecycleStop; @@ -46,8 +46,8 @@ public class SupervisorManager private final MetadataSupervisorManager metadataSupervisorManager; private final ConcurrentHashMap> supervisors = new ConcurrentHashMap<>(); - // SupervisorTaskAutoscaler could be null - private final ConcurrentHashMap autoscalers = new ConcurrentHashMap<>(); + // SupervisorTaskAutoScaler could be null + private final ConcurrentHashMap autoscalers = new ConcurrentHashMap<>(); private final Object lock = new Object(); private volatile boolean started = false; @@ -149,7 +149,7 @@ public void stop() for (String id : supervisors.keySet()) { try { supervisors.get(id).lhs.stop(false); - SupervisorTaskAutoscaler autoscaler = autoscalers.get(id); + SupervisorTaskAutoScaler autoscaler = autoscalers.get(id); if (autoscaler != null) { autoscaler.stop(); } @@ -201,7 +201,7 @@ public boolean resetSupervisor(String id, @Nullable DataSourceMetadata dataSourc } supervisor.lhs.reset(dataSourceMetadata); - SupervisorTaskAutoscaler autoscaler = autoscalers.get(id); + SupervisorTaskAutoScaler autoscaler = autoscalers.get(id); if (autoscaler != null) { autoscaler.reset(); } @@ -256,7 +256,7 @@ private boolean possiblyStopAndRemoveSupervisorInternal(String id, boolean write pair.lhs.stop(true); supervisors.remove(id); - SupervisorTaskAutoscaler autoscler = autoscalers.get(id); + SupervisorTaskAutoScaler autoscler = autoscalers.get(id); if (autoscler != null) { autoscler.stop(); autoscalers.remove(id); @@ -306,7 +306,7 @@ private boolean createAndStartSupervisorInternal(SupervisorSpec spec, boolean pe } Supervisor supervisor; - SupervisorTaskAutoscaler autoscaler; + SupervisorTaskAutoScaler autoscaler; try { supervisor = spec.createSupervisor(); autoscaler = spec.createAutoscaler(supervisor); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index e63db4465daa..1c7a177d6dd1 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -92,7 +92,6 @@ import javax.annotation.Nullable; import javax.validation.constraints.NotNull; - import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -335,7 +334,7 @@ private class DynamicAllocationTasksNotice implements Notice } /** - * This method will do lags points collection and check dynamic scale action is necessary or not. + * This method will do lag points collection and check dynamic scale action is necessary or not. */ @Override public void handle() @@ -363,15 +362,15 @@ public void handle() return; } } - if (nowTime - dynamicTriggerLastRunTime < autoScalerConfig.getMinTriggerDynamicFrequencyMillis()) { + if (nowTime - dynamicTriggerLastRunTime < autoScalerConfig.getMinTriggerScaleActionFrequencyMillis()) { log.info( "DynamicAllocationTasksNotice submitted again in [%d] millis, minTriggerDynamicFrequency is [%s] for dataSource [%s], skipping it!", - nowTime - dynamicTriggerLastRunTime, autoScalerConfig.getMinTriggerDynamicFrequencyMillis(), dataSource + nowTime - dynamicTriggerLastRunTime, autoScalerConfig.getMinTriggerScaleActionFrequencyMillis(), dataSource ); return; } - Integer desriedTaskCount = scaleAction.call(); - boolean allocationSuccess = dynamicAllocate(desriedTaskCount); + final Integer desriedTaskCount = scaleAction.call(); + boolean allocationSuccess = changeTaskCount(desriedTaskCount); if (allocationSuccess) { dynamicTriggerLastRunTime = nowTime; } @@ -387,17 +386,17 @@ public void handle() * This method determines how to do scale actions based on collected lag points. * If scale action is triggered : * First of all, call gracefulShutdownInternal() which will change the state of current datasource ingest tasks from reading to publishing. - * Secondly, clear all the stateful data structures: activelyReadingTaskGroups, partitionGroups, partitionOffsets, pendingCompletionTaskGroups, partitionIds. These structures will be rebuild in the next 'RunNotice'. + * Secondly, clear all the stateful data structures: activelyReadingTaskGroups, partitionGroups, partitionOffsets, pendingCompletionTaskGroups, partitionIds. These structures will be rebuiled in the next 'RunNotice'. * Finally, change the taskCount in SeekableStreamSupervisorIOConfig and sync it to MetadataStorage. * After the taskCount is changed in SeekableStreamSupervisorIOConfig, next RunNotice will create scaled number of ingest tasks without resubmitting the supervisor. - * @param desiredActiveTaskCount desired taskCount compute from autoscaler - * @return Boolean flag indicating if scale action was executed or not. If true, it will wait at least 'minTriggerDynamicFrequency' before next 'dynamicAllocate'. - * If false, it will do 'dynamicAllocate' again after 'dynamicCheckPeriod' millis. + * @param desiredActiveTaskCount desired taskCount computed from AutoScaler + * @return Boolean flag indicating if scale action was executed or not. If true, it will wait at least 'minTriggerScaleActionFrequencyMillis' before next 'changeTaskCount'. + * If false, it will do 'changeTaskCount' again after 'scaleActionPeriodMillis' millis. * @throws InterruptedException * @throws ExecutionException * @throws TimeoutException */ - private boolean dynamicAllocate(Integer desiredActiveTaskCount) throws InterruptedException, ExecutionException, TimeoutException + private boolean changeTaskCount(Integer desiredActiveTaskCount) throws InterruptedException, ExecutionException, TimeoutException { int currentActiveTaskCount; Collection activeTaskGroups = activelyReadingTaskGroups.values(); @@ -406,14 +405,13 @@ private boolean dynamicAllocate(Integer desiredActiveTaskCount) throws Interrupt if (desiredActiveTaskCount == -1 || desiredActiveTaskCount == currentActiveTaskCount) { return false; } else { - log.debug( + log.info( "Starting scale action, current active task count is [%d] and desired task count is [%d] for dataSource [%s].", currentActiveTaskCount, desiredActiveTaskCount, dataSource ); gracefulShutdownInternal(); changeTaskCountInIOConfig(desiredActiveTaskCount); - // clear everything - clearAllocationInfos(); + clearAllocationInfo(); log.info("Changed taskCount to [%s] for dataSource [%s].", desiredActiveTaskCount, dataSource); return true; } @@ -428,15 +426,15 @@ private void changeTaskCountInIOConfig(int desiredActiveTaskCount) MetadataSupervisorManager metadataSupervisorManager = supervisorManager.get().getMetadataSupervisorManager(); metadataSupervisorManager.insert(dataSource, spec); } else { - log.warn("supervisorManager is null in taskMaster, skipping scale action for dataSource [%s].", dataSource); + log.error("supervisorManager is null in taskMaster, skipping scale action for dataSource [%s].", dataSource); } } catch (Exception e) { - log.warn("Failed to sync taskCount to MetaStorage for dataSource [%s].", dataSource); + log.error("supervisorManager is null in taskMaster, skipping scale action for dataSource [%s].", dataSource); } } - private void clearAllocationInfos() + private void clearAllocationInfo() { activelyReadingTaskGroups.clear(); partitionGroups.clear(); @@ -664,7 +662,7 @@ public SeekableStreamSupervisor( int workerThreads; int chatThreads; - if (autoScalerConfig != null && autoScalerConfig.getEnableTaskAutoscaler()) { + if (autoScalerConfig != null && autoScalerConfig.getEnableTaskAutoScaler()) { log.info("Running Task autoscaler for datasource [%s]", dataSource); workerThreads = (this.tuningConfig.getWorkerThreads() != null diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java index df0ee848d21f..3ed55ec1ec4f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorIOConfig.java @@ -47,7 +47,7 @@ public abstract class SeekableStreamSupervisorIOConfig private final Optional lateMessageRejectionPeriod; private final Optional earlyMessageRejectionPeriod; private final Optional lateMessageRejectionStartDateTime; - @Nullable private final AutoScalerConfig autoscalerConfig; + @Nullable private final AutoScalerConfig autoScalerConfig; public SeekableStreamSupervisorIOConfig( String stream, @@ -61,7 +61,7 @@ public SeekableStreamSupervisorIOConfig( Period completionTimeout, Period lateMessageRejectionPeriod, Period earlyMessageRejectionPeriod, - @Nullable AutoScalerConfig autoscalerConfig, + @Nullable AutoScalerConfig autoScalerConfig, DateTime lateMessageRejectionStartDateTime ) { @@ -69,10 +69,10 @@ public SeekableStreamSupervisorIOConfig( this.inputFormat = inputFormat; this.replicas = replicas != null ? replicas : 1; // Could be null - this.autoscalerConfig = autoscalerConfig; + this.autoScalerConfig = autoScalerConfig; // if autoscaler is enable then taskcount will be ignored here. and init taskcount will be equal to taskCountMin - if (autoscalerConfig != null && autoscalerConfig.getEnableTaskAutoscaler()) { - this.taskCount = autoscalerConfig.getTaskCountMin(); + if (autoScalerConfig != null && autoScalerConfig.getEnableTaskAutoScaler()) { + this.taskCount = autoScalerConfig.getTaskCountMin(); } else { this.taskCount = taskCount != null ? taskCount : 1; } @@ -127,7 +127,7 @@ public Integer getReplicas() @JsonProperty public AutoScalerConfig getAutoscalerConfig() { - return autoscalerConfig; + return autoScalerConfig; } @JsonProperty diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java index f4c9417fb953..638769891c1d 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java @@ -25,7 +25,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import org.apache.druid.annotations.SuppressFBWarnings; import org.apache.druid.guice.annotations.Json; import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator; import org.apache.druid.indexing.overlord.TaskMaster; @@ -33,12 +32,10 @@ import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; import org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientFactory; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScaler; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DummyAutoScaler; -import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.NoopTaskAutoScaler; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.segment.incremental.RowIngestionMetersFactory; import org.apache.druid.segment.indexing.DataSchema; @@ -163,29 +160,13 @@ public DruidMonitorSchedulerConfig getMonitorSchedulerConfig() * @return autoScaler, disable autoscale will return dummyAutoScaler and enable autoscale wiil return defaultAutoScaler by default. */ @Override - @SuppressFBWarnings(value = "RV_RETURN_VALUE_IGNORED", justification = "using siwtch(String)") - public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + public SupervisorTaskAutoScaler createAutoscaler(Supervisor supervisor) { - String dataSource = getId(); - SupervisorTaskAutoscaler autoScaler = new DummyAutoScaler(supervisor, dataSource); AutoScalerConfig autoScalerConfig = ingestionSchema.getIOConfig().getAutoscalerConfig(); - - // kinesis'autoscalerConfig is always null for now, So that kinesis will hold a DummyAutoScaler. - // only SeekableStreamSupervisor is supported here. - if (autoScalerConfig != null - && autoScalerConfig.getEnableTaskAutoscaler() - && supervisor instanceof SeekableStreamSupervisor) { - - String autoScalerStrategy = autoScalerConfig.getAutoScalerStrategy(); - - // will thorw 'Return value of String.hashCode() ignored : RV_RETURN_VALUE_IGNORED' just Suppress it. - switch (StringUtils.toLowerCase(autoScalerStrategy)) { - default: { - autoScaler = new DefaultAutoScaler(supervisor, dataSource, autoScalerConfig, this); - } - } + if (autoScalerConfig != null && autoScalerConfig.getEnableTaskAutoScaler() && supervisor instanceof SeekableStreamSupervisor) { + return autoScalerConfig.createAutoScaler(supervisor, this); } - return autoScaler; + return new NoopTaskAutoScaler(); } @Override diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java index ca3553dda28d..53174a17bbac 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/AutoScalerConfig.java @@ -23,18 +23,21 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.apache.druid.guice.annotations.UnstableApi; +import org.apache.druid.indexing.overlord.supervisor.Supervisor; +import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; @UnstableApi -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "autoScalerStrategy", defaultImpl = DefaultAutoScalerConfig.class) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "autoScalerStrategy", defaultImpl = LagBasedAutoScalerConfig.class) @JsonSubTypes(value = { - @Type(name = "default", value = DefaultAutoScalerConfig.class) + @Type(name = "lagBased", value = LagBasedAutoScalerConfig.class) }) public interface AutoScalerConfig { - boolean getEnableTaskAutoscaler(); - long getMinTriggerDynamicFrequencyMillis(); - String getAutoScalerStrategy(); + boolean getEnableTaskAutoScaler(); + long getMinTriggerScaleActionFrequencyMillis(); int getTaskCountMax(); int getTaskCountMin(); + SupervisorTaskAutoScaler createAutoScaler(Supervisor supervisor, SupervisorSpec spec); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java deleted file mode 100644 index 031825a37183..000000000000 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScaler.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.indexing.seekablestream.supervisor.autoscaler; - -import org.apache.commons.collections4.queue.CircularFifoQueue; -import org.apache.druid.indexing.overlord.supervisor.Supervisor; -import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; -import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisor; -import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.java.util.common.concurrent.Execs; -import org.apache.druid.java.util.emitter.EmittingLogger; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; - -public class DefaultAutoScaler implements SupervisorTaskAutoscaler -{ - private static final EmittingLogger log = new EmittingLogger(DefaultAutoScaler.class); - private final String dataSource; - private final CircularFifoQueue lagMetricsQueue; - private final ScheduledExecutorService lagComputationExec; - private final ScheduledExecutorService allocationExec; - private final SupervisorSpec spec; - private final SeekableStreamSupervisor supervisor; - private final DefaultAutoScalerConfig defaultAutoScalerConfig; - - private static ReentrantLock lock = new ReentrantLock(true); - - - public DefaultAutoScaler(Supervisor supervisor, String dataSource, AutoScalerConfig autoScalerConfig, SupervisorSpec spec) - { - this.defaultAutoScalerConfig = (DefaultAutoScalerConfig) autoScalerConfig; - String supervisorId = StringUtils.format("KafkaSupervisor-%s", dataSource); - this.dataSource = dataSource; - int slots = (int) (defaultAutoScalerConfig.getMetricsCollectionRangeMillis() / defaultAutoScalerConfig.getMetricsCollectionIntervalMillis()) + 1; - log.debug(" The interval of metrics collection is [%s], [%s] timeRange will collect [%s] data points for dataSource [%s].", defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), defaultAutoScalerConfig.getMetricsCollectionRangeMillis(), slots, dataSource); - this.lagMetricsQueue = new CircularFifoQueue<>(slots); - this.allocationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Allocation-%d"); - this.lagComputationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Computation-%d"); - this.spec = spec; - this.supervisor = (SeekableStreamSupervisor) supervisor; - } - - @Override - public void start() - { - Callable scaleAction = new Callable() { - @Override - public Integer call() - { - lock.lock(); - int desireTaskCount = -1; - try { - desireTaskCount = computeDesireTaskCount(new ArrayList<>(lagMetricsQueue)); - - if (desireTaskCount != -1) { - lagMetricsQueue.clear(); - } - } - catch (Exception ex) { - log.warn(ex, "Exception when computeDesireTaskCount [%s]", dataSource); - } - finally { - lock.unlock(); - } - return desireTaskCount; - } - }; - - log.info("enableTaskAutoscaler for datasource [%s]", dataSource); - log.debug("Collect and compute lags at fixed rate of [%s] for dataSource[%s].", defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), dataSource); - lagComputationExec.scheduleAtFixedRate( - collectAndComputeLags(), - defaultAutoScalerConfig.getDynamicCheckStartDelayMillis(), // wait for tasks to start up - defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), - TimeUnit.MILLISECONDS - ); - log.debug("allocate task at fixed rate of [%s], dataSource [%s].", defaultAutoScalerConfig.getDynamicCheckPeriod(), dataSource); - allocationExec.scheduleAtFixedRate( - supervisor.buildDynamicAllocationTask(scaleAction), - defaultAutoScalerConfig.getDynamicCheckStartDelayMillis() + defaultAutoScalerConfig.getMetricsCollectionRangeMillis(), - defaultAutoScalerConfig.getDynamicCheckPeriod(), - TimeUnit.MILLISECONDS - ); - } - - @Override - public void stop() - { - allocationExec.shutdownNow(); - lagComputationExec.shutdownNow(); - } - - @Override - public void reset() - { - // clear queue for kafka lags - if (lagMetricsQueue != null) { - try { - lock.lock(); - lagMetricsQueue.clear(); - } - catch (Exception e) { - log.warn(e, "Error,when clear queue in rest action"); - } - finally { - lock.unlock(); - } - } - } - - /** - * This method compute current consume lags. Get the total lags of all partition and fill in lagMetricsQueue - * @return a Runnbale object to do collect and compute action. - */ - private Runnable collectAndComputeLags() - { - return new Runnable() { - @Override - public void run() - { - lock.lock(); - try { - if (!spec.isSuspended()) { - LagStats lagStats = supervisor.computeLagStats(); - if (lagStats == null) { - lagMetricsQueue.offer(0L); - } else { - long totalLags = lagStats.getTotalLag(); - lagMetricsQueue.offer(totalLags > 0 ? totalLags : 0L); - } - - log.debug("Current lag metric points [%s] for dataSource [%s].", new ArrayList<>(lagMetricsQueue), dataSource); - } else { - log.debug("[%s] supervisor is suspended, skip to collect kafka lags", dataSource); - } - } - catch (Exception e) { - log.warn(e, "Error, When collect lags"); - } - finally { - lock.unlock(); - } - } - }; - } - - /** - * This method determines whether to do scale actions based on collected lag points. - * Current algorithm of scale is simple: - * First of all, compute the proportion of lag points higher/lower than scaleOutThreshold/scaleInThreshold, getting scaleOutThreshold/scaleInThreshold. - * Secondly, compare scaleOutThreshold/scaleInThreshold with triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency. P.S. Scale out action has higher priority than scale in action. - * Finaly, if scaleOutThreshold/scaleInThreshold is higher than triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency, scale out/in action would be triggered. - * @param lags the lag metrics of Stream(Kafka/Kinesis) - * @return Boolean flag, do scale action successfully or not. If true , it will take at least 'minTriggerDynamicFrequency' before next 'dynamicAllocatie'. - * If false, it will do 'dynamicAllocate' again after 'dynamicCheckPeriod'. - * - * @return Integer. target number of tasksCount, -1 means skip scale action. - */ - private Integer computeDesireTaskCount(List lags) - { - // if supervisor is not suspended, ensure required tasks are running - // if suspended, ensure tasks have been requested to gracefully stop - log.info("[%s] supervisor is running, start to check dynamic allocate task logic. Current collected lags : [%s]", dataSource, lags); - int beyond = 0; - int within = 0; - int metricsCount = lags.size(); - for (Long lag : lags) { - if (lag >= defaultAutoScalerConfig.getScaleOutThreshold()) { - beyond++; - } - if (lag <= defaultAutoScalerConfig.getScaleInThreshold()) { - within++; - } - } - double beyondProportion = beyond * 1.0 / metricsCount; - double withinProportion = within * 1.0 / metricsCount; - log.debug("triggerScaleOutThresholdFrequency is [%s] and triggerScaleInThresholdFrequency is [%s] for dataSource [%s].", defaultAutoScalerConfig.getTriggerScaleOutThresholdFrequency(), defaultAutoScalerConfig.getTriggerScaleInThresholdFrequency(), dataSource); - log.info("beyondProportion is [%s] and withinProportion is [%s] for dataSource [%s].", beyondProportion, withinProportion, dataSource); - - int currentActiveTaskCount = supervisor.getActiveTaskGroupsCount(); - if (currentActiveTaskCount < 0) { - log.info("CurrentActiveTaskCount is lower than 0 ??? skip [%s].", dataSource); - return -1; - } - int desiredActiveTaskCount; - - if (beyondProportion >= defaultAutoScalerConfig.getTriggerScaleOutThresholdFrequency()) { - // Do Scale out - int taskCount = currentActiveTaskCount + defaultAutoScalerConfig.getScaleOutStep(); - if (currentActiveTaskCount == defaultAutoScalerConfig.getTaskCountMax()) { - log.info("CurrentActiveTaskCount reach task count Max limit, skip to scale out tasks for dataSource [%s].", dataSource); - return -1; - } else { - desiredActiveTaskCount = Math.min(taskCount, defaultAutoScalerConfig.getTaskCountMax()); - } - - return desiredActiveTaskCount; - } - - if (withinProportion >= defaultAutoScalerConfig.getTriggerScaleInThresholdFrequency()) { - // Do Scale in - int taskCount = currentActiveTaskCount - defaultAutoScalerConfig.getScaleInStep(); - if (currentActiveTaskCount == defaultAutoScalerConfig.getTaskCountMin()) { - log.info("CurrentActiveTaskCount reach task count Min limit, skip to scale in tasks for dataSource [%s].", dataSource); - return -1; - } else { - desiredActiveTaskCount = Math.max(taskCount, defaultAutoScalerConfig.getTaskCountMin()); - } - log.debug("Start to scale in tasks, current active task number [%s] and desire task number is [%s] for dataSource [%s].", currentActiveTaskCount, desiredActiveTaskCount, dataSource); - return desiredActiveTaskCount; - } - - return -1; - } - - public DefaultAutoScalerConfig getAutoScalerConfig() - { - return defaultAutoScalerConfig; - } -} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java new file mode 100644 index 000000000000..ceccc9ec20ba --- /dev/null +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.indexing.seekablestream.supervisor.autoscaler; + +import org.apache.commons.collections4.queue.CircularFifoQueue; +import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; +import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisor; +import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.emitter.EmittingLogger; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +public class LagBasedAutoScaler implements SupervisorTaskAutoScaler +{ + private static final EmittingLogger log = new EmittingLogger(LagBasedAutoScaler.class); + private final String dataSource; + private final CircularFifoQueue lagMetricsQueue; + private final ScheduledExecutorService lagComputationExec; + private final ScheduledExecutorService allocationExec; + private final SupervisorSpec spec; + private final SeekableStreamSupervisor supervisor; + private final LagBasedAutoScalerConfig lagBasedAutoScalerConfig; + + private static final ReentrantLock LOCK = new ReentrantLock(true); + + public LagBasedAutoScaler(SeekableStreamSupervisor supervisor, String dataSource, + LagBasedAutoScalerConfig autoScalerConfig, SupervisorSpec spec + ) + { + this.lagBasedAutoScalerConfig = autoScalerConfig; + final String supervisorId = StringUtils.format("Supervisor-%s", dataSource); + this.dataSource = dataSource; + final int slots = (int) (lagBasedAutoScalerConfig.getLagCollectionRangeMillis() / lagBasedAutoScalerConfig + .getLagCollectionIntervalMillis()) + 1; + this.lagMetricsQueue = new CircularFifoQueue<>(slots); + this.allocationExec = Execs.scheduledSingleThreaded(supervisorId + "-Allocation-%d"); + this.lagComputationExec = Execs.scheduledSingleThreaded(supervisorId + "-Computation-%d"); + this.spec = spec; + this.supervisor = supervisor; + } + + @Override + public void start() + { + Callable scaleAction = () -> { + LOCK.lock(); + int desiredTaskCount = -1; + try { + desiredTaskCount = computeDesiredTaskCount(new ArrayList<>(lagMetricsQueue)); + + if (desiredTaskCount != -1) { + lagMetricsQueue.clear(); + } + } + catch (Exception ex) { + log.warn(ex, "Exception while computing desired task count for [%s]", dataSource); + } + finally { + LOCK.unlock(); + } + return desiredTaskCount; + }; + + lagComputationExec.scheduleAtFixedRate( + computeAndCollectLag(), + lagBasedAutoScalerConfig.getScaleActionStartDelayMillis(), // wait for tasks to start up + lagBasedAutoScalerConfig.getLagCollectionIntervalMillis(), + TimeUnit.MILLISECONDS + ); + allocationExec.scheduleAtFixedRate( + supervisor.buildDynamicAllocationTask(scaleAction), + lagBasedAutoScalerConfig.getScaleActionStartDelayMillis() + lagBasedAutoScalerConfig + .getLagCollectionRangeMillis(), + lagBasedAutoScalerConfig.getScaleActionPeriodMillis(), + TimeUnit.MILLISECONDS + ); + log.info( + "LagBasedAutoScaler will collect lag every [%d] millis and will keep [%d] data points for the last [%d] millis for dataSource [%s]", + lagBasedAutoScalerConfig.getLagCollectionIntervalMillis(), lagMetricsQueue.size(), + lagBasedAutoScalerConfig.getLagCollectionRangeMillis(), dataSource + ); + } + + @Override + public void stop() + { + allocationExec.shutdownNow(); + lagComputationExec.shutdownNow(); + } + + @Override + public void reset() + { + // clear queue for kafka lags + if (lagMetricsQueue != null) { + try { + LOCK.lock(); + lagMetricsQueue.clear(); + } + catch (Exception e) { + log.warn(e, "Error,when clear queue in rest action"); + } + finally { + LOCK.unlock(); + } + } + } + + /** + * This method computes current consumer lag. Gets the total lag of all partitions and fill in the lagMetricsQueue + * + * @return a Runnbale object to compute and collect lag. + */ + private Runnable computeAndCollectLag() + { + return () -> { + LOCK.lock(); + try { + if (!spec.isSuspended()) { + LagStats lagStats = supervisor.computeLagStats(); + if (lagStats == null) { + lagMetricsQueue.offer(0L); + } else { + long totalLags = lagStats.getTotalLag(); + lagMetricsQueue.offer(totalLags > 0 ? totalLags : 0L); + } + log.debug("Current lags [%s] for dataSource [%s].", new ArrayList<>(lagMetricsQueue), dataSource); + } else { + log.warn("[%s] supervisor is suspended, skipping lag collection", dataSource); + } + } + catch (Exception e) { + log.error(e, "Error while collecting lags"); + } + finally { + LOCK.unlock(); + } + }; + } + + /** + * This method determines whether to do scale actions based on collected lag points. + * Current algorithm of scale is simple: + * First of all, compute the proportion of lag points higher/lower than scaleOutThreshold/scaleInThreshold, getting scaleOutThreshold/scaleInThreshold. + * Secondly, compare scaleOutThreshold/scaleInThreshold with triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency. P.S. Scale out action has higher priority than scale in action. + * Finaly, if scaleOutThreshold/scaleInThreshold is higher than triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency, scale out/in action would be triggered. + * + * @param lags the lag metrics of Stream(Kafka/Kinesis) + * @return Integer. target number of tasksCount, -1 means skip scale action. + */ + private Integer computeDesiredTaskCount(List lags) + { + // if supervisor is not suspended, ensure required tasks are running + // if suspended, ensure tasks have been requested to gracefully stop + log.debug("Computing desired task count for [%s], based on following lags : [%s]", dataSource, lags); + int beyond = 0; + int within = 0; + int metricsCount = lags.size(); + for (Long lag : lags) { + if (lag >= lagBasedAutoScalerConfig.getScaleOutThreshold()) { + beyond++; + } + if (lag <= lagBasedAutoScalerConfig.getScaleInThreshold()) { + within++; + } + } + double beyondProportion = beyond * 1.0 / metricsCount; + double withinProportion = within * 1.0 / metricsCount; + + log.debug("Calculated beyondProportion is [%s] and withinProportion is [%s] for dataSource [%s].", beyondProportion, + withinProportion, dataSource + ); + + int currentActiveTaskCount = supervisor.getActiveTaskGroupsCount(); + if (currentActiveTaskCount < 0) { + log.warn("CurrentActiveTaskCount is lower than 0 ? skipping computation of desired task count for [%s].", + dataSource + ); + return -1; + } + int desiredActiveTaskCount; + + if (beyondProportion >= lagBasedAutoScalerConfig.getTriggerScaleOutThresholdFrequency()) { + // Do Scale out + int taskCount = currentActiveTaskCount + lagBasedAutoScalerConfig.getScaleOutStep(); + if (currentActiveTaskCount == lagBasedAutoScalerConfig.getTaskCountMax()) { + log.warn("CurrentActiveTaskCount reached task count Max limit, skipping scale out action for dataSource [%s].", + dataSource + ); + return -1; + } else { + desiredActiveTaskCount = Math.min(taskCount, lagBasedAutoScalerConfig.getTaskCountMax()); + } + return desiredActiveTaskCount; + } + + if (withinProportion >= lagBasedAutoScalerConfig.getTriggerScaleInThresholdFrequency()) { + // Do Scale in + int taskCount = currentActiveTaskCount - lagBasedAutoScalerConfig.getScaleInStep(); + if (currentActiveTaskCount == lagBasedAutoScalerConfig.getTaskCountMin()) { + log.warn("CurrentActiveTaskCount reached task count Min limit, skipping scale in action for dataSource [%s].", + dataSource + ); + return -1; + } else { + desiredActiveTaskCount = Math.max(taskCount, lagBasedAutoScalerConfig.getTaskCountMin()); + } + return desiredActiveTaskCount; + } + return -1; + } + + public LagBasedAutoScalerConfig getAutoScalerConfig() + { + return lagBasedAutoScalerConfig; + } +} diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScalerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScalerConfig.java similarity index 60% rename from indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScalerConfig.java rename to indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScalerConfig.java index f4169529a49e..26516618450e 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DefaultAutoScalerConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScalerConfig.java @@ -21,15 +21,19 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.indexing.overlord.supervisor.Supervisor; +import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; +import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisor; import javax.annotation.Nullable; -public class DefaultAutoScalerConfig implements AutoScalerConfig +public class LagBasedAutoScalerConfig implements AutoScalerConfig { - private final long metricsCollectionIntervalMillis; - private final long metricsCollectionRangeMillis; - private final long dynamicCheckStartDelayMillis; - private final long dynamicCheckPeriod; + private final long lagCollectionIntervalMillis; + private final long lagCollectionRangeMillis; + private final long scaleActionStartDelayMillis; + private final long scaleActionPeriodMillis; private final long scaleOutThreshold; private final long scaleInThreshold; private final double triggerScaleOutThresholdFrequency; @@ -38,16 +42,15 @@ public class DefaultAutoScalerConfig implements AutoScalerConfig private int taskCountMin; private final int scaleInStep; private final int scaleOutStep; - private final boolean enableTaskAutoscaler; - private final String autoScalerStrategy; - private final long minTriggerDynamicFrequencyMillis; + private final boolean enableTaskAutoScaler; + private final long minTriggerScaleActionFrequencyMillis; @JsonCreator - public DefaultAutoScalerConfig( - @Nullable @JsonProperty("metricsCollectionIntervalMillis") Long metricsCollectionIntervalMillis, - @Nullable @JsonProperty("metricsCollectionRangeMillis") Long metricsCollectionRangeMillis, - @Nullable @JsonProperty("dynamicCheckStartDelayMillis") Long dynamicCheckStartDelayMillis, - @Nullable @JsonProperty("dynamicCheckPeriod") Long dynamicCheckPeriod, + public LagBasedAutoScalerConfig( + @Nullable @JsonProperty("lagCollectionIntervalMillis") Long lagCollectionIntervalMillis, + @Nullable @JsonProperty("lagCollectionRangeMillis") Long lagCollectionRangeMillis, + @Nullable @JsonProperty("scaleActionStartDelayMillis") Long scaleActionStartDelayMillis, + @Nullable @JsonProperty("scaleActionPeriodMillis") Long scaleActionPeriodMillis, @Nullable @JsonProperty("scaleOutThreshold") Long scaleOutThreshold, @Nullable @JsonProperty("scaleInThreshold") Long scaleInThreshold, @Nullable @JsonProperty("triggerScaleOutThresholdFrequency") Double triggerScaleOutThresholdFrequency, @@ -56,15 +59,15 @@ public DefaultAutoScalerConfig( @JsonProperty("taskCountMin") Integer taskCountMin, @Nullable @JsonProperty("scaleInStep") Integer scaleInStep, @Nullable @JsonProperty("scaleOutStep") Integer scaleOutStep, - @Nullable @JsonProperty("enableTaskAutoscaler") Boolean enableTaskAutoscaler, - @Nullable @JsonProperty("autoScalerStrategy") String autoScalerStrategy, - @Nullable @JsonProperty("minTriggerDynamicFrequencyMillis") Long minTriggerDynamicFrequencyMillis) - { - this.enableTaskAutoscaler = enableTaskAutoscaler != null ? enableTaskAutoscaler : false; - this.metricsCollectionIntervalMillis = metricsCollectionIntervalMillis != null ? metricsCollectionIntervalMillis : 30000; - this.metricsCollectionRangeMillis = metricsCollectionRangeMillis != null ? metricsCollectionRangeMillis : 600000; - this.dynamicCheckStartDelayMillis = dynamicCheckStartDelayMillis != null ? dynamicCheckStartDelayMillis : 300000; - this.dynamicCheckPeriod = dynamicCheckPeriod != null ? dynamicCheckPeriod : 60000; + @Nullable @JsonProperty("enableTaskAutoScaler") Boolean enableTaskAutoScaler, + @Nullable @JsonProperty("minTriggerScaleActionFrequencyMillis") Long minTriggerScaleActionFrequencyMillis + ) + { + this.enableTaskAutoScaler = enableTaskAutoScaler != null ? enableTaskAutoScaler : false; + this.lagCollectionIntervalMillis = lagCollectionIntervalMillis != null ? lagCollectionIntervalMillis : 30000; + this.lagCollectionRangeMillis = lagCollectionRangeMillis != null ? lagCollectionRangeMillis : 600000; + this.scaleActionStartDelayMillis = scaleActionStartDelayMillis != null ? scaleActionStartDelayMillis : 300000; + this.scaleActionPeriodMillis = scaleActionPeriodMillis != null ? scaleActionPeriodMillis : 60000; this.scaleOutThreshold = scaleOutThreshold != null ? scaleOutThreshold : 6000000; this.scaleInThreshold = scaleInThreshold != null ? scaleInThreshold : 1000000; this.triggerScaleOutThresholdFrequency = triggerScaleOutThresholdFrequency != null ? triggerScaleOutThresholdFrequency : 0.3; @@ -72,7 +75,7 @@ public DefaultAutoScalerConfig( // Only do taskCountMax and taskCountMin check when autoscaler is enabled. So that users left autoConfig empty{} will not throw any exception and autoscaler is disabled. // If autoscaler is disabled, no matter what configs are set, they are not used. - if (this.enableTaskAutoscaler) { + if (this.enableTaskAutoScaler) { if (taskCountMax == null || taskCountMin == null) { throw new RuntimeException("taskCountMax or taskCountMin can't be null!"); } else if (taskCountMax < taskCountMin) { @@ -84,32 +87,32 @@ public DefaultAutoScalerConfig( this.scaleInStep = scaleInStep != null ? scaleInStep : 1; this.scaleOutStep = scaleOutStep != null ? scaleOutStep : 2; - this.autoScalerStrategy = autoScalerStrategy != null ? autoScalerStrategy : "default"; - this.minTriggerDynamicFrequencyMillis = minTriggerDynamicFrequencyMillis != null ? minTriggerDynamicFrequencyMillis : 600000; + this.minTriggerScaleActionFrequencyMillis = minTriggerScaleActionFrequencyMillis + != null ? minTriggerScaleActionFrequencyMillis : 600000; } @JsonProperty - public long getMetricsCollectionIntervalMillis() + public long getLagCollectionIntervalMillis() { - return metricsCollectionIntervalMillis; + return lagCollectionIntervalMillis; } @JsonProperty - public long getMetricsCollectionRangeMillis() + public long getLagCollectionRangeMillis() { - return metricsCollectionRangeMillis; + return lagCollectionRangeMillis; } @JsonProperty - public long getDynamicCheckStartDelayMillis() + public long getScaleActionStartDelayMillis() { - return dynamicCheckStartDelayMillis; + return scaleActionStartDelayMillis; } @JsonProperty - public long getDynamicCheckPeriod() + public long getScaleActionPeriodMillis() { - return dynamicCheckPeriod; + return scaleActionPeriodMillis; } @JsonProperty @@ -150,6 +153,12 @@ public int getTaskCountMin() return taskCountMin; } + @Override + public SupervisorTaskAutoScaler createAutoScaler(Supervisor supervisor, SupervisorSpec spec) + { + return new LagBasedAutoScaler((SeekableStreamSupervisor) supervisor, spec.getId(), this, spec); + } + @JsonProperty public int getScaleInStep() { @@ -164,22 +173,15 @@ public int getScaleOutStep() @Override @JsonProperty - public boolean getEnableTaskAutoscaler() - { - return enableTaskAutoscaler; - } - - @Override - @JsonProperty - public String getAutoScalerStrategy() + public boolean getEnableTaskAutoScaler() { - return autoScalerStrategy; + return enableTaskAutoScaler; } @Override @JsonProperty - public long getMinTriggerDynamicFrequencyMillis() + public long getMinTriggerScaleActionFrequencyMillis() { - return minTriggerDynamicFrequencyMillis; + return minTriggerScaleActionFrequencyMillis; } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DummyAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/NoopTaskAutoScaler.java similarity index 83% rename from indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DummyAutoScaler.java rename to indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/NoopTaskAutoScaler.java index 2d779e09c33e..9bf41e5b0be2 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/DummyAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/NoopTaskAutoScaler.java @@ -19,12 +19,11 @@ package org.apache.druid.indexing.seekablestream.supervisor.autoscaler; -import org.apache.druid.indexing.overlord.supervisor.Supervisor; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; -public class DummyAutoScaler implements SupervisorTaskAutoscaler +public class NoopTaskAutoScaler implements SupervisorTaskAutoScaler { - public DummyAutoScaler(Supervisor supervisor, String dataSource) + public NoopTaskAutoScaler() { } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java index 062908b17085..3b775ca94915 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/security/OverlordSecurityResourceFilterTest.java @@ -32,8 +32,8 @@ import org.apache.druid.indexing.overlord.supervisor.SupervisorManager; import org.apache.druid.indexing.overlord.supervisor.SupervisorResource; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DummyAutoScaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.NoopTaskAutoScaler; import org.apache.druid.indexing.worker.http.WorkerResource; import org.apache.druid.server.http.security.ResourceFilterTestHelper; import org.apache.druid.server.security.AuthorizerMapper; @@ -129,9 +129,9 @@ public Supervisor createSupervisor() } @Override - public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + public SupervisorTaskAutoScaler createAutoscaler(Supervisor supervisor) { - return new DummyAutoScaler(null, null); + return new NoopTaskAutoScaler(); } @Override diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorResourceTest.java index e714798f01b5..c22460ec9269 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorResourceTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/supervisor/SupervisorResourceTest.java @@ -26,7 +26,7 @@ import com.google.common.collect.ImmutableSet; import org.apache.druid.indexing.overlord.DataSourceMetadata; import org.apache.druid.indexing.overlord.TaskMaster; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.segment.TestHelper; import org.apache.druid.server.security.Access; @@ -1157,7 +1157,7 @@ public Supervisor createSupervisor() } @Override - public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + public SupervisorTaskAutoScaler createAutoscaler(Supervisor supervisor) { return null; } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 4190454722fa..5c302a7d7b3a 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -37,7 +37,7 @@ import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig; import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; import org.apache.druid.indexing.seekablestream.common.OrderedSequenceNumber; import org.apache.druid.indexing.seekablestream.common.RecordSupplier; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisor; @@ -47,9 +47,9 @@ import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorSpec; import org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisorTuningConfig; import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScaler; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DefaultAutoScalerConfig; -import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.DummyAutoScaler; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.LagBasedAutoScaler; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.LagBasedAutoScalerConfig; +import org.apache.druid.indexing.seekablestream.supervisor.autoscaler.NoopTaskAutoScaler; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.parsers.JSONPathSpec; @@ -344,12 +344,12 @@ public LagStats computeLagStats() } - private static class TesstSeekableStreamSupervisorSpec extends SeekableStreamSupervisorSpec + private static class TestSeekableStreamSupervisorSpec extends SeekableStreamSupervisorSpec { private SeekableStreamSupervisor supervisor; private String id; - public TesstSeekableStreamSupervisorSpec(SeekableStreamSupervisorIngestionSpec ingestionSchema, + public TestSeekableStreamSupervisorSpec(SeekableStreamSupervisorIngestionSpec ingestionSchema, @Nullable Map context, Boolean suspended, TaskStorage taskStorage, @@ -512,23 +512,23 @@ public String toString() public void testAutoScalerConfig() { AutoScalerConfig autoScalerConfigEmpty = mapper.convertValue(new HashMap<>(), AutoScalerConfig.class); - Assert.assertTrue(autoScalerConfigEmpty instanceof DefaultAutoScalerConfig); - Assert.assertFalse(autoScalerConfigEmpty.getEnableTaskAutoscaler()); + Assert.assertTrue(autoScalerConfigEmpty instanceof LagBasedAutoScalerConfig); + Assert.assertFalse(autoScalerConfigEmpty.getEnableTaskAutoScaler()); AutoScalerConfig autoScalerConfigNull = mapper.convertValue(null, AutoScalerConfig.class); Assert.assertNull(autoScalerConfigNull); - AutoScalerConfig autoScalerConfigDefault = mapper.convertValue(ImmutableMap.of("autoScalerStrategy", "default"), AutoScalerConfig.class); - Assert.assertTrue(autoScalerConfigDefault instanceof DefaultAutoScalerConfig); + AutoScalerConfig autoScalerConfigDefault = mapper.convertValue(ImmutableMap.of("autoScalerStrategy", "lagBased"), AutoScalerConfig.class); + Assert.assertTrue(autoScalerConfigDefault instanceof LagBasedAutoScalerConfig); - AutoScalerConfig autoScalerConfigValue = mapper.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", "1"), AutoScalerConfig.class); - Assert.assertTrue(autoScalerConfigValue instanceof DefaultAutoScalerConfig); - DefaultAutoScalerConfig defaultAutoScalerConfig = (DefaultAutoScalerConfig) autoScalerConfigValue; - Assert.assertEquals(defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), 1); + AutoScalerConfig autoScalerConfigValue = mapper.convertValue(ImmutableMap.of("lagCollectionIntervalMillis", "1"), AutoScalerConfig.class); + Assert.assertTrue(autoScalerConfigValue instanceof LagBasedAutoScalerConfig); + LagBasedAutoScalerConfig lagBasedAutoScalerConfig = (LagBasedAutoScalerConfig) autoScalerConfigValue; + Assert.assertEquals(lagBasedAutoScalerConfig.getLagCollectionIntervalMillis(), 1); Exception e = null; try { - AutoScalerConfig autoScalerError = mapper.convertValue(ImmutableMap.of("enableTaskAutoscaler", "true", "taskCountMax", "1", "taskCountMin", "4"), AutoScalerConfig.class); + AutoScalerConfig autoScalerError = mapper.convertValue(ImmutableMap.of("enableTaskAutoScaler", "true", "taskCountMax", "1", "taskCountMin", "4"), AutoScalerConfig.class); } catch (RuntimeException ex) { e = ex; @@ -538,7 +538,7 @@ public void testAutoScalerConfig() e = null; try { // taskCountMax and taskCountMin couldn't be ignored. - AutoScalerConfig autoScalerError2 = mapper.convertValue(ImmutableMap.of("enableTaskAutoscaler", "true"), AutoScalerConfig.class); + AutoScalerConfig autoScalerError2 = mapper.convertValue(ImmutableMap.of("enableTaskAutoScaler", "true"), AutoScalerConfig.class); } catch (RuntimeException ex) { e = ex; @@ -551,34 +551,34 @@ public void testAutoScalerConfig() @Test public void testAutoScalerCreated() { - HashMap autoscalerConfig = new HashMap<>(); - autoscalerConfig.put("enableTaskAutoscaler", true); - autoscalerConfig.put("metricsCollectionIntervalMillis", 500); - autoscalerConfig.put("metricsCollectionRangeMillis", 500); - autoscalerConfig.put("scaleOutThreshold", 5000000); - autoscalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); - autoscalerConfig.put("scaleInThreshold", 1000000); - autoscalerConfig.put("triggerScaleInThresholdFrequency", 0.8); - autoscalerConfig.put("dynamicCheckStartDelayMillis", 0); - autoscalerConfig.put("dynamicCheckPeriod", 100); - autoscalerConfig.put("taskCountMax", 8); - autoscalerConfig.put("taskCountMin", 1); - autoscalerConfig.put("scaleInStep", 1); - autoscalerConfig.put("scaleOutStep", 2); - autoscalerConfig.put("minTriggerDynamicFrequencyMillis", 1200000); + HashMap autoScalerConfig = new HashMap<>(); + autoScalerConfig.put("enableTaskAutoScaler", true); + autoScalerConfig.put("lagCollectionIntervalMillis", 500); + autoScalerConfig.put("lagCollectionRangeMillis", 500); + autoScalerConfig.put("scaleOutThreshold", 5000000); + autoScalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); + autoScalerConfig.put("scaleInThreshold", 1000000); + autoScalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoScalerConfig.put("scaleActionStartDelayMillis", 0); + autoScalerConfig.put("scaleActionPeriodMillis", 100); + autoScalerConfig.put("taskCountMax", 8); + autoScalerConfig.put("taskCountMin", 1); + autoScalerConfig.put("scaleInStep", 1); + autoScalerConfig.put("scaleOutStep", 2); + autoScalerConfig.put("minTriggerScaleActionFrequencyMillis", 1200000); EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes(); EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes(); EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, AutoScalerConfig.class)).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); EasyMock.replay(supervisor4); - TesstSeekableStreamSupervisorSpec spec = new TesstSeekableStreamSupervisorSpec(ingestionSchema, + TestSeekableStreamSupervisorSpec spec = new TestSeekableStreamSupervisorSpec(ingestionSchema, null, false, taskStorage, @@ -592,30 +592,30 @@ public void testAutoScalerCreated() supervisorStateManagerConfig, supervisor4, "id1"); - SupervisorTaskAutoscaler autoscaler = spec.createAutoscaler(supervisor4); - Assert.assertTrue(autoscaler instanceof DefaultAutoScaler); + SupervisorTaskAutoScaler autoscaler = spec.createAutoscaler(supervisor4); + Assert.assertTrue(autoscaler instanceof LagBasedAutoScaler); EasyMock.reset(seekableStreamSupervisorIOConfig); - autoscalerConfig.put("enableTaskAutoscaler", false); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, AutoScalerConfig.class)).anyTimes(); + autoScalerConfig.put("enableTaskAutoScaler", false); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); - SupervisorTaskAutoscaler autoscaler2 = spec.createAutoscaler(supervisor4); - Assert.assertTrue(autoscaler2 instanceof DummyAutoScaler); + SupervisorTaskAutoScaler autoscaler2 = spec.createAutoscaler(supervisor4); + Assert.assertTrue(autoscaler2 instanceof NoopTaskAutoScaler); EasyMock.reset(seekableStreamSupervisorIOConfig); - autoscalerConfig.remove("enableTaskAutoscaler"); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, AutoScalerConfig.class)).anyTimes(); + autoScalerConfig.remove("enableTaskAutoScaler"); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); - SupervisorTaskAutoscaler autoscaler3 = spec.createAutoscaler(supervisor4); - Assert.assertTrue(autoscaler3 instanceof DummyAutoScaler); + SupervisorTaskAutoScaler autoscaler3 = spec.createAutoscaler(supervisor4); + Assert.assertTrue(autoscaler3 instanceof NoopTaskAutoScaler); EasyMock.reset(seekableStreamSupervisorIOConfig); - autoscalerConfig.clear(); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoscalerConfig, AutoScalerConfig.class)).anyTimes(); + autoScalerConfig.clear(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); - Assert.assertTrue(autoscalerConfig.isEmpty()); - SupervisorTaskAutoscaler autoscaler4 = spec.createAutoscaler(supervisor4); - Assert.assertTrue(autoscaler4 instanceof DummyAutoScaler); + Assert.assertTrue(autoScalerConfig.isEmpty()); + SupervisorTaskAutoScaler autoscaler4 = spec.createAutoscaler(supervisor4); + Assert.assertTrue(autoscaler4 instanceof NoopTaskAutoScaler); } @@ -627,13 +627,13 @@ public void testDefaultAutoScalerConfigCreatedWithDefault() EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); EasyMock.replay(ingestionSchema); - EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(ImmutableMap.of("metricsCollectionIntervalMillis", "1", "enableTaskAutoscaler", true, "taskCountMax", "4", "taskCountMin", "1"), AutoScalerConfig.class)).anyTimes(); + EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(ImmutableMap.of("lagCollectionIntervalMillis", "1", "enableTaskAutoScaler", true, "taskCountMax", "4", "taskCountMin", "1"), AutoScalerConfig.class)).anyTimes(); EasyMock.replay(seekableStreamSupervisorIOConfig); EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes(); EasyMock.replay(supervisor4); - TesstSeekableStreamSupervisorSpec spec = new TesstSeekableStreamSupervisorSpec(ingestionSchema, + TestSeekableStreamSupervisorSpec spec = new TestSeekableStreamSupervisorSpec(ingestionSchema, null, false, taskStorage, @@ -647,22 +647,21 @@ public void testDefaultAutoScalerConfigCreatedWithDefault() supervisorStateManagerConfig, supervisor4, "id1"); - SupervisorTaskAutoscaler autoscaler = spec.createAutoscaler(supervisor4); - Assert.assertTrue(autoscaler instanceof DefaultAutoScaler); - DefaultAutoScaler defaultAutoScaler = (DefaultAutoScaler) autoscaler; - DefaultAutoScalerConfig defaultAutoScalerConfig = defaultAutoScaler.getAutoScalerConfig(); - Assert.assertEquals(defaultAutoScalerConfig.getMetricsCollectionIntervalMillis(), 1); - Assert.assertEquals(defaultAutoScalerConfig.getMetricsCollectionRangeMillis(), 600000); - Assert.assertEquals(defaultAutoScalerConfig.getDynamicCheckStartDelayMillis(), 300000); - Assert.assertEquals(defaultAutoScalerConfig.getDynamicCheckPeriod(), 60000); - Assert.assertEquals(defaultAutoScalerConfig.getScaleOutThreshold(), 6000000); - Assert.assertEquals(defaultAutoScalerConfig.getScaleInThreshold(), 1000000); - Assert.assertEquals(defaultAutoScalerConfig.getTaskCountMax(), 4); - Assert.assertEquals(defaultAutoScalerConfig.getTaskCountMin(), 1); - Assert.assertEquals(defaultAutoScalerConfig.getScaleInStep(), 1); - Assert.assertEquals(defaultAutoScalerConfig.getScaleOutStep(), 2); - Assert.assertEquals(defaultAutoScalerConfig.getAutoScalerStrategy(), "default"); - Assert.assertEquals(defaultAutoScalerConfig.getMinTriggerDynamicFrequencyMillis(), 600000); + SupervisorTaskAutoScaler autoscaler = spec.createAutoscaler(supervisor4); + Assert.assertTrue(autoscaler instanceof LagBasedAutoScaler); + LagBasedAutoScaler lagBasedAutoScaler = (LagBasedAutoScaler) autoscaler; + LagBasedAutoScalerConfig lagBasedAutoScalerConfig = lagBasedAutoScaler.getAutoScalerConfig(); + Assert.assertEquals(lagBasedAutoScalerConfig.getLagCollectionIntervalMillis(), 1); + Assert.assertEquals(lagBasedAutoScalerConfig.getLagCollectionRangeMillis(), 600000); + Assert.assertEquals(lagBasedAutoScalerConfig.getScaleActionStartDelayMillis(), 300000); + Assert.assertEquals(lagBasedAutoScalerConfig.getScaleActionPeriodMillis(), 60000); + Assert.assertEquals(lagBasedAutoScalerConfig.getScaleOutThreshold(), 6000000); + Assert.assertEquals(lagBasedAutoScalerConfig.getScaleInThreshold(), 1000000); + Assert.assertEquals(lagBasedAutoScalerConfig.getTaskCountMax(), 4); + Assert.assertEquals(lagBasedAutoScalerConfig.getTaskCountMin(), 1); + Assert.assertEquals(lagBasedAutoScalerConfig.getScaleInStep(), 1); + Assert.assertEquals(lagBasedAutoScalerConfig.getScaleOutStep(), 2); + Assert.assertEquals(lagBasedAutoScalerConfig.getMinTriggerScaleActionFrequencyMillis(), 600000); } @Test @@ -691,7 +690,7 @@ public void testSeekableStreamSupervisorSpecWithScaleOut() throws InterruptedExc LagStats lagStats = supervisor.computeLagStats(); - DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(), AutoScalerConfig.class), spec); + LagBasedAutoScaler autoScaler = new LagBasedAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(), LagBasedAutoScalerConfig.class), spec); supervisor.start(); autoScaler.start(); supervisor.runInternal(); @@ -729,7 +728,7 @@ public void testSeekableStreamSupervisorSpecWithScaleIn() throws InterruptedExce EasyMock.replay(taskMaster); TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); - DefaultAutoScaler autoScaler = new DefaultAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleInProperties(), AutoScalerConfig.class), spec); + LagBasedAutoScaler autoScaler = new LagBasedAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleInProperties(), LagBasedAutoScalerConfig.class), spec); // enable autoscaler so that taskcount config will be ignored and init value of taskCount will use taskCountMin. Assert.assertEquals(1, (int) supervisor.getIoConfig().getTaskCount()); @@ -783,7 +782,7 @@ public void testSeekableStreamSupervisorSpecWithScaleDisable() throws Interrupte EasyMock.replay(taskMaster); TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); - DummyAutoScaler autoScaler = new DummyAutoScaler(supervisor, DATASOURCE); + NoopTaskAutoScaler autoScaler = new NoopTaskAutoScaler(); supervisor.start(); autoScaler.start(); supervisor.runInternal(); @@ -856,42 +855,42 @@ private SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boolean scal private static Map getScaleOutProperties() { - HashMap autoscalerConfig = new HashMap<>(); - autoscalerConfig.put("enableTaskAutoscaler", true); - autoscalerConfig.put("metricsCollectionIntervalMillis", 500); - autoscalerConfig.put("metricsCollectionRangeMillis", 500); - autoscalerConfig.put("scaleOutThreshold", 0); - autoscalerConfig.put("triggerScaleOutThresholdFrequency", 0.0); - autoscalerConfig.put("scaleInThreshold", 1000000); - autoscalerConfig.put("triggerScaleInThresholdFrequency", 0.8); - autoscalerConfig.put("dynamicCheckStartDelayMillis", 0); - autoscalerConfig.put("dynamicCheckPeriod", 100); - autoscalerConfig.put("taskCountMax", 2); - autoscalerConfig.put("taskCountMin", 1); - autoscalerConfig.put("scaleInStep", 1); - autoscalerConfig.put("scaleOutStep", 2); - autoscalerConfig.put("minTriggerDynamicFrequencyMillis", 1200000); - return autoscalerConfig; + HashMap autoScalerConfig = new HashMap<>(); + autoScalerConfig.put("enableTaskAutoScaler", true); + autoScalerConfig.put("lagCollectionIntervalMillis", 500); + autoScalerConfig.put("lagCollectionRangeMillis", 500); + autoScalerConfig.put("scaleOutThreshold", 0); + autoScalerConfig.put("triggerScaleOutThresholdFrequency", 0.0); + autoScalerConfig.put("scaleInThreshold", 1000000); + autoScalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoScalerConfig.put("scaleActionStartDelayMillis", 0); + autoScalerConfig.put("scaleActionPeriodMillis", 100); + autoScalerConfig.put("taskCountMax", 2); + autoScalerConfig.put("taskCountMin", 1); + autoScalerConfig.put("scaleInStep", 1); + autoScalerConfig.put("scaleOutStep", 2); + autoScalerConfig.put("minTriggerScaleActionFrequencyMillis", 1200000); + return autoScalerConfig; } private static Map getScaleInProperties() { - HashMap autoscalerConfig = new HashMap<>(); - autoscalerConfig.put("enableTaskAutoscaler", true); - autoscalerConfig.put("metricsCollectionIntervalMillis", 500); - autoscalerConfig.put("metricsCollectionRangeMillis", 500); - autoscalerConfig.put("scaleOutThreshold", 8000000); - autoscalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); - autoscalerConfig.put("scaleInThreshold", 0); - autoscalerConfig.put("triggerScaleInThresholdFrequency", 0.0); - autoscalerConfig.put("dynamicCheckStartDelayMillis", 0); - autoscalerConfig.put("dynamicCheckPeriod", 100); - autoscalerConfig.put("taskCountMax", 2); - autoscalerConfig.put("taskCountMin", 1); - autoscalerConfig.put("scaleInStep", 1); - autoscalerConfig.put("scaleOutStep", 2); - autoscalerConfig.put("minTriggerDynamicFrequencyMillis", 1200000); - return autoscalerConfig; + HashMap autoScalerConfig = new HashMap<>(); + autoScalerConfig.put("enableTaskAutoScaler", true); + autoScalerConfig.put("lagCollectionIntervalMillis", 500); + autoScalerConfig.put("lagCollectionRangeMillis", 500); + autoScalerConfig.put("scaleOutThreshold", 8000000); + autoScalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); + autoScalerConfig.put("scaleInThreshold", 0); + autoScalerConfig.put("triggerScaleInThresholdFrequency", 0.0); + autoScalerConfig.put("scaleActionStartDelayMillis", 0); + autoScalerConfig.put("scaleActionPeriodMillis", 100); + autoScalerConfig.put("taskCountMax", 2); + autoScalerConfig.put("taskCountMin", 1); + autoScalerConfig.put("scaleInStep", 1); + autoScalerConfig.put("scaleOutStep", 2); + autoScalerConfig.put("minTriggerScaleActionFrequencyMillis", 1200000); + return autoScalerConfig; } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index 58dfcae1c54c..62b20a0eb9e6 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -838,22 +838,22 @@ private static SeekableStreamSupervisorIOConfig getIOConfig() private static Map getProperties() { - HashMap autoscalerConfig = new HashMap<>(); - autoscalerConfig.put("enableTaskAutoscaler", true); - autoscalerConfig.put("metricsCollectionIntervalMillis", 500); - autoscalerConfig.put("metricsCollectionRangeMillis", 500); - autoscalerConfig.put("scaleOutThreshold", 5000000); - autoscalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); - autoscalerConfig.put("scaleInThreshold", 1000000); - autoscalerConfig.put("triggerScaleInThresholdFrequency", 0.8); - autoscalerConfig.put("dynamicCheckStartDelayMillis", 0); - autoscalerConfig.put("dynamicCheckPeriod", 100); - autoscalerConfig.put("taskCountMax", 8); - autoscalerConfig.put("taskCountMin", 1); - autoscalerConfig.put("scaleInStep", 1); - autoscalerConfig.put("scaleOutStep", 2); - autoscalerConfig.put("minTriggerDynamicFrequencyMillis", 1200000); - return autoscalerConfig; + HashMap autoScalerConfig = new HashMap<>(); + autoScalerConfig.put("enableTaskAutoScaler", true); + autoScalerConfig.put("lagCollectionIntervalMillis", 500); + autoScalerConfig.put("lagCollectionRangeMillis", 500); + autoScalerConfig.put("scaleOutThreshold", 5000000); + autoScalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); + autoScalerConfig.put("scaleInThreshold", 1000000); + autoScalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoScalerConfig.put("scaleActionStartDelayMillis", 0); + autoScalerConfig.put("scaleActionPeriodMillis", 100); + autoScalerConfig.put("taskCountMax", 8); + autoScalerConfig.put("taskCountMin", 1); + autoScalerConfig.put("scaleInStep", 1); + autoScalerConfig.put("scaleOutStep", 2); + autoScalerConfig.put("minTriggerScaleActionFrequencyMillis", 1200000); + return autoScalerConfig; } private static SeekableStreamSupervisorTuningConfig getTuningConfig() diff --git a/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json b/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json index 49428cceec48..ee32c9d22fc5 100644 --- a/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json +++ b/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json @@ -48,21 +48,21 @@ "ioConfig": { "%%TOPIC_KEY%%": "%%TOPIC_VALUE%%", "%%STREAM_PROPERTIES_KEY%%": %%STREAM_PROPERTIES_VALUE%%, - "autoscalerConfig": { - "enableTaskAutoscaler": true, - "metricsCollectionIntervalMillis": 500, - "metricsCollectionRangeMillis": 500, + "autoScalerConfig": { + "enableTaskAutoScaler": true, + "lagCollectionIntervalMillis": 500, + "lagCollectionRangeMillis": 500, "scaleOutThreshold": 0, "triggerScaleOutThresholdFrequency": 0.0, "scaleInThreshold": 1000000, "triggerScaleInThresholdFrequency": 0.9, - "dynamicCheckStartDelayMillis": 0, - "dynamicCheckPeriod": 100, + "scaleActionStartDelayMillis": 0, + "scaleActionPeriodMillis": 100, "taskCountMax": 2, "taskCountMin": 1, "scaleInStep": 1, "scaleOutStep": 2, - "minTriggerDynamicFrequencyMillis": 600000 + "minTriggerScaleActionFrequencyMillis": 600000 }, "taskCount": 1, "replicas": 1, diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java index b345163b2c32..66d11399bd26 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/Supervisor.java @@ -67,8 +67,8 @@ default Boolean isHealthy() void checkpoint(int taskGroupId, DataSourceMetadata checkpointMetadata); /** - * Collect maxLag, totalLag, avgLag - * Only support Kafka ingestion so far. + * Computes maxLag, totalLag and avgLag + * Only supports Kafka ingestion so far. */ LagStats computeLagStats(); diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java index 5b5137a4f76f..9b44cd08dd12 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/SupervisorSpec.java @@ -21,7 +21,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; import java.util.List; @@ -41,7 +41,7 @@ public interface SupervisorSpec */ Supervisor createSupervisor(); - default SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + default SupervisorTaskAutoScaler createAutoscaler(Supervisor supervisor) { return null; } diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/SupervisorTaskAutoscaler.java b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/SupervisorTaskAutoScaler.java similarity index 95% rename from server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/SupervisorTaskAutoscaler.java rename to server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/SupervisorTaskAutoScaler.java index ecc1d2810316..c921e2740b87 100644 --- a/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/SupervisorTaskAutoscaler.java +++ b/server/src/main/java/org/apache/druid/indexing/overlord/supervisor/autoscaler/SupervisorTaskAutoScaler.java @@ -19,7 +19,7 @@ package org.apache.druid.indexing.overlord.supervisor.autoscaler; -public interface SupervisorTaskAutoscaler +public interface SupervisorTaskAutoScaler { void start(); void stop(); diff --git a/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java b/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java index ad7f9699bd72..fd5fac09e51c 100644 --- a/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java +++ b/server/src/test/java/org/apache/druid/indexing/NoopSupervisorSpecTest.java @@ -22,7 +22,7 @@ import org.apache.druid.indexing.overlord.supervisor.NoopSupervisorSpec; import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.autoscaler.LagStats; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; import org.junit.Assert; import org.junit.Test; @@ -38,7 +38,7 @@ public void testNoopSupervisorSpecWithAutoscaler() try { NoopSupervisorSpec noopSupervisorSpec = new NoopSupervisorSpec(null, Collections.singletonList("datasource1")); Supervisor supervisor = noopSupervisorSpec.createSupervisor(); - SupervisorTaskAutoscaler autoscaler = noopSupervisorSpec.createAutoscaler(supervisor); + SupervisorTaskAutoScaler autoscaler = noopSupervisorSpec.createAutoscaler(supervisor); Assert.assertNull(autoscaler); Callable noop = new Callable() { @Override diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java index 89f8b6a05ad4..5c40757f61a5 100644 --- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java +++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataSupervisorManagerTest.java @@ -25,7 +25,7 @@ import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; import org.apache.druid.indexing.overlord.supervisor.VersionedSupervisorSpec; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.StringUtils; import org.junit.After; @@ -187,7 +187,7 @@ public Supervisor createSupervisor() } @Override - public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + public SupervisorTaskAutoScaler createAutoscaler(Supervisor supervisor) { return null; } diff --git a/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java b/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java index 9eb3b5435ad9..ffacfa26b8ba 100644 --- a/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java +++ b/server/src/test/java/org/apache/druid/metadata/TestSupervisorSpec.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.indexing.overlord.supervisor.Supervisor; import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec; -import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoscaler; +import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler; import java.util.List; import java.util.Objects; @@ -54,7 +54,7 @@ public Supervisor createSupervisor() } @Override - public SupervisorTaskAutoscaler createAutoscaler(Supervisor supervisor) + public SupervisorTaskAutoScaler createAutoscaler(Supervisor supervisor) { return null; } From 1f1008266a0040a74f9bd8c0deffddf923d67d9e Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 25 Feb 2021 19:21:37 +0800 Subject: [PATCH 43/47] log changed --- .../seekablestream/supervisor/SeekableStreamSupervisor.java | 2 +- .../supervisor/SeekableStreamSupervisorSpec.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 1c7a177d6dd1..7db219b727a0 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -430,7 +430,7 @@ private void changeTaskCountInIOConfig(int desiredActiveTaskCount) } } catch (Exception e) { - log.error("supervisorManager is null in taskMaster, skipping scale action for dataSource [%s].", dataSource); + log.error(e, "Failed to sync taskCount to MetaStorage for dataSource [%s].", dataSource); } } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java index 638769891c1d..ff1d31756bbe 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorSpec.java @@ -155,9 +155,9 @@ public DruidMonitorSchedulerConfig getMonitorSchedulerConfig() public abstract Supervisor createSupervisor(); /** - * need to notice that autoScaler would be null which means autoscale is dissable. + * An autoScaler instance will be returned depending on the autoScalerConfig. In case autoScalerConfig is null or autoScaler is disabled then NoopTaskAutoScaler will be returned. * @param supervisor - * @return autoScaler, disable autoscale will return dummyAutoScaler and enable autoscale wiil return defaultAutoScaler by default. + * @return autoScaler */ @Override public SupervisorTaskAutoScaler createAutoscaler(Supervisor supervisor) From 6334e2ba0ddf48ddfd884fcdfe9361cee1431e6b Mon Sep 17 00:00:00 2001 From: yuezhang Date: Thu, 25 Feb 2021 21:13:20 +0800 Subject: [PATCH 44/47] do StringUtils.encodeForFormat when create allocationExec --- .../supervisor/autoscaler/LagBasedAutoScaler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java index ceccc9ec20ba..9a82648a8ef0 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java @@ -58,8 +58,8 @@ public LagBasedAutoScaler(SeekableStreamSupervisor supervisor, String dataSource final int slots = (int) (lagBasedAutoScalerConfig.getLagCollectionRangeMillis() / lagBasedAutoScalerConfig .getLagCollectionIntervalMillis()) + 1; this.lagMetricsQueue = new CircularFifoQueue<>(slots); - this.allocationExec = Execs.scheduledSingleThreaded(supervisorId + "-Allocation-%d"); - this.lagComputationExec = Execs.scheduledSingleThreaded(supervisorId + "-Computation-%d"); + this.allocationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Allocation-%d"); + this.lagComputationExec = Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId) + "-Computation-%d"); this.spec = spec; this.supervisor = supervisor; } From 22339ddc83976758809570dd1c92d7506c26fcfa Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 2 Mar 2021 14:06:57 +0800 Subject: [PATCH 45/47] code review && limit taskCountMax to partitionNumbers --- .../extensions-core/kafka-ingestion.md | 105 ++++++++++++++++-- .../kafka/supervisor/KafkaSupervisorTest.java | 4 +- .../supervisor/SeekableStreamSupervisor.java | 9 +- .../autoscaler/LagBasedAutoScaler.java | 28 ++--- .../autoscaler/LagBasedAutoScalerConfig.java | 20 ++-- .../SeekableStreamSupervisorSpecTest.java | 80 ++++++++++--- .../SeekableStreamSupervisorStateTest.java | 4 +- ...ervisor_with_autoscaler_spec_template.json | 4 +- 8 files changed, 202 insertions(+), 52 deletions(-) diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 810b940b40d8..88a1cf57ed95 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -146,26 +146,115 @@ A sample supervisor spec is shown below: |`lateMessageRejectionStartDateTime`|ISO8601 DateTime|Configure tasks to reject messages with timestamps earlier than this date time; for example if this is set to `2016-01-01T11:00Z` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline).|no (default == none)| |`lateMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps earlier than this period before the task was created; for example if this is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps earlier than *2016-01-01T11:00Z* will be dropped. This may help prevent concurrency issues if your data stream has late messages and you have multiple pipelines that need to operate on the same segments (e.g. a realtime and a nightly batch ingestion pipeline). Please note that only one of `lateMessageRejectionPeriod` or `lateMessageRejectionStartDateTime` can be specified.|no (default == none)| |`earlyMessageRejectionPeriod`|ISO8601 Period|Configure tasks to reject messages with timestamps later than this period after the task reached its taskDuration; for example if this is set to `PT1H`, the taskDuration is set to `PT1H` and the supervisor creates a task at *2016-01-01T12:00Z*, messages with timestamps later than *2016-01-01T14:00Z* will be dropped. **Note:** Tasks sometimes run past their task duration, for example, in cases of supervisor failover. Setting earlyMessageRejectionPeriod too low may cause messages to be dropped unexpectedly whenever a task runs past its originally configured task duration.|no (default == none)| -|`autoScalerConfig`|Object|`autoScalerConfig` to specify how to auto scale the number of Kafka ingest tasks based on Lag metrics. ONLY supported for Kafka indexing as of now. See [Tasks Autoscaler Properties](#Task Autoscaler Properties) for details.|no (default == null)| +|`autoScalerConfig`|Object|`autoScalerConfig` to specify how to auto scale the number of Kafka ingest tasks. ONLY supported for Kafka indexing as of now. See [Tasks Autoscaler Properties](#Task Autoscaler Properties) for details.|no (default == null)| ### Task Autoscaler Properties + +> Note that Task AutoScaler is currently designated as experimental. + | Property | Description | Required | | ------------- | ------------- | ------------- | | `enableTaskAutoScaler` | Whether enable this feature or not. Set false or ignored here will disable `autoScaler` even though `autoScalerConfig` is not null| no (default == false) | -| `lagCollectionIntervalMillis` | Define the frequency of lag points collection. | no (default == 30000) | +| `taskCountMax` | Maximum value of task count. Make Sure `taskCountMax >= taskCountMin`. If `taskCountMax > {numKafkaPartitions}`, the maximum number of reading tasks would be equal to `{numKafkaPartitions}` and `taskCountMax` would be ignored. | yes | +| `taskCountMin` | Minimum value of task count. When enable autoscaler, the value of taskCount in `IOConfig` will be ignored, and `taskCountMin` will be the number of tasks that ingestion starts going up to `taskCountMax`| yes | +| `minTriggerScaleActionFrequencyMillis` | Minimum time interval between two scale actions | no (default == 600000) | +| `autoScalerStrategy` | The algorithm of `autoScaler`. ONLY `lagBased` is supported for now. See [Lag Based AutoScaler Strategy Related Properties](#LagBased AutoScalerStrategy Related Properties) for details.| no (default == `lagBased`) | + +### Lag Based AutoScaler Strategy Related Properties +| Property | Description | Required | +| ------------- | ------------- | ------------- | +| `lagCollectionIntervalMillis` | Period of lag points collection. | no (default == 30000) | | `lagCollectionRangeMillis` | The total time window of lag collection, Use with `lagCollectionIntervalMillis`，it means that in the recent `lagCollectionRangeMillis`, collect lag metric points every `lagCollectionIntervalMillis`. | no (default == 600000) | | `scaleOutThreshold` | The Threshold of scale out action | no (default == 6000000) | -| `triggerScaleOutThresholdFrequency` | If `triggerScaleOutThresholdFrequency` percent of lag points are higher than `scaleOutThreshold`, then do scale out action. | no (default == 0.3) | +| `triggerScaleOutFractionThreshold` | If `triggerScaleOutFractionThreshold` percent of lag points are higher than `scaleOutThreshold`, then do scale out action. | no (default == 0.3) | | `scaleInThreshold` | The Threshold of scale in action | no (default == 1000000) | -| `triggerScaleInThresholdFrequency` | If `triggerScaleInThresholdFrequency` percent of lag points are lower than `scaleOutThreshold`, then do scale in action. | no (default == 0.9) | +| `triggerScaleInFractionThreshold` | If `triggerScaleInFractionThreshold` percent of lag points are lower than `scaleOutThreshold`, then do scale in action. | no (default == 0.9) | | `scaleActionStartDelayMillis` | Number of milliseconds after supervisor starts when first check scale logic. | no (default == 300000) | | `scaleActionPeriodMillis` | The frequency of checking whether to do scale action in millis | no (default == 60000) | -| `taskCountMax` | Maximum value of task count. Make Sure `taskCountMax >= taskCountMin` | yes | -| `taskCountMin` | Minimum value of task count. When enable autoscaler, the value of taskCount in `IOConfig` will be ignored, and `taskCountMin` will be the number of tasks that ingestion starts going up to `taskCountMax`| yes | | `scaleInStep` | How many tasks to reduce at a time | no (default == 1) | | `scaleOutStep` | How many tasks to add at a time | no (default == 2) | -| `minTriggerScaleActionFrequencyMillis` | Minimum time interval between two scale actions | no (default == 600000) | -| `autoScalerStrategy` | The algorithm of `autoScaler`. ONLY `lagBased` is supported for now. | no (default == `lagBased`) | + +A sample supervisor spec with `lagBased` autoScaler enabled is shown below: +```json +{ + "type": "kafka", + "dataSchema": { + "dataSource": "metrics-kafka", + "timestampSpec": { + "column": "timestamp", + "format": "auto" + }, + "dimensionsSpec": { + "dimensions": [ + + ], + "dimensionExclusions": [ + "timestamp", + "value" + ] + }, + "metricsSpec": [ + { + "name": "count", + "type": "count" + }, + { + "name": "value_sum", + "fieldName": "value", + "type": "doubleSum" + }, + { + "name": "value_min", + "fieldName": "value", + "type": "doubleMin" + }, + { + "name": "value_max", + "fieldName": "value", + "type": "doubleMax" + } + ], + "granularitySpec": { + "type": "uniform", + "segmentGranularity": "HOUR", + "queryGranularity": "NONE" + } + }, + "ioConfig": { + "topic": "metrics", + "inputFormat": { + "type": "json" + }, + "consumerProperties": { + "bootstrap.servers": "localhost:9092" + }, + "autoScalerConfig": { + "enableTaskAutoScaler": true, + "taskCountMax": 6, + "taskCountMin": 2, + "minTriggerScaleActionFrequencyMillis": 600000, + "autoScalerStrategy": "lagBased", + "lagCollectionIntervalMillis": 30000, + "lagCollectionRangeMillis": 600000, + "scaleOutThreshold": 6000000, + "triggerScaleOutFractionThreshold": 0.3, + "scaleInThreshold": 1000000, + "triggerScaleInFractionThreshold": 0.9, + "scaleActionStartDelayMillis": 300000, + "scaleActionPeriodMillis": 60000, + "scaleInStep": 1, + "scaleOutStep": 2 + }, + "taskCount":1, + "replicas":1, + "taskDuration":"PT1H" + }, + "tuningConfig":{ + "type":"kafka", + "maxRowsPerSegment":5000000 + } +} +``` #### More on consumerProperties diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java index 547acf510294..ac6b6e50590f 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorTest.java @@ -266,9 +266,9 @@ public KafkaIndexTaskClient build( autoScalerConfig.put("lagCollectionIntervalMillis", 500); autoScalerConfig.put("lagCollectionRangeMillis", 500); autoScalerConfig.put("scaleOutThreshold", 0); - autoScalerConfig.put("triggerScaleOutThresholdFrequency", 0.0); + autoScalerConfig.put("triggerScaleOutFractionThreshold", 0.0); autoScalerConfig.put("scaleInThreshold", 1000000); - autoScalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoScalerConfig.put("triggerScaleInFractionThreshold", 0.8); autoScalerConfig.put("scaleActionStartDelayMillis", 0); autoScalerConfig.put("scaleActionPeriodMillis", 100); autoScalerConfig.put("taskCountMax", 2); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 7db219b727a0..468363dcbe84 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -396,13 +396,13 @@ public void handle() * @throws ExecutionException * @throws TimeoutException */ - private boolean changeTaskCount(Integer desiredActiveTaskCount) throws InterruptedException, ExecutionException, TimeoutException + private boolean changeTaskCount(int desiredActiveTaskCount) throws InterruptedException, ExecutionException, TimeoutException { int currentActiveTaskCount; Collection activeTaskGroups = activelyReadingTaskGroups.values(); currentActiveTaskCount = activeTaskGroups.size(); - if (desiredActiveTaskCount == -1 || desiredActiveTaskCount == currentActiveTaskCount) { + if (desiredActiveTaskCount < 0 || desiredActiveTaskCount == currentActiveTaskCount) { return false; } else { log.info( @@ -2058,6 +2058,11 @@ protected boolean supportsPartitionExpiration() return false; } + public int getPartitionNumbers() + { + return recordSupplier.getPartitionIds(ioConfig.getStream()).size(); + } + private boolean updatePartitionDataFromStream() { List previousPartitionIds = new ArrayList<>(partitionIds); diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java index 9a82648a8ef0..a0fe1af924dd 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java @@ -167,13 +167,13 @@ private Runnable computeAndCollectLag() * This method determines whether to do scale actions based on collected lag points. * Current algorithm of scale is simple: * First of all, compute the proportion of lag points higher/lower than scaleOutThreshold/scaleInThreshold, getting scaleOutThreshold/scaleInThreshold. - * Secondly, compare scaleOutThreshold/scaleInThreshold with triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency. P.S. Scale out action has higher priority than scale in action. - * Finaly, if scaleOutThreshold/scaleInThreshold is higher than triggerScaleOutThresholdFrequency/triggerScaleInThresholdFrequency, scale out/in action would be triggered. + * Secondly, compare scaleOutThreshold/scaleInThreshold with triggerScaleOutFractionThreshold/triggerScaleInFractionThreshold. P.S. Scale out action has higher priority than scale in action. + * Finaly, if scaleOutThreshold/scaleInThreshold is higher than triggerScaleOutFractionThreshold/triggerScaleInFractionThreshold, scale out/in action would be triggered. * * @param lags the lag metrics of Stream(Kafka/Kinesis) * @return Integer. target number of tasksCount, -1 means skip scale action. */ - private Integer computeDesiredTaskCount(List lags) + private int computeDesiredTaskCount(List lags) { // if supervisor is not suspended, ensure required tasks are running // if suspended, ensure tasks have been requested to gracefully stop @@ -197,29 +197,31 @@ private Integer computeDesiredTaskCount(List lags) ); int currentActiveTaskCount = supervisor.getActiveTaskGroupsCount(); - if (currentActiveTaskCount < 0) { - log.warn("CurrentActiveTaskCount is lower than 0 ? skipping computation of desired task count for [%s].", - dataSource - ); - return -1; - } int desiredActiveTaskCount; - if (beyondProportion >= lagBasedAutoScalerConfig.getTriggerScaleOutThresholdFrequency()) { + if (beyondProportion >= lagBasedAutoScalerConfig.getTriggerScaleOutFractionThreshold()) { // Do Scale out int taskCount = currentActiveTaskCount + lagBasedAutoScalerConfig.getScaleOutStep(); - if (currentActiveTaskCount == lagBasedAutoScalerConfig.getTaskCountMax()) { + + int partitionNumbers = supervisor.getPartitionNumbers(); + if (partitionNumbers <= 0) { + log.warn("Partition number for [%s] <= 0 ? how can it be?", dataSource); + return -1; + } + + int actualTaskCountMax = Math.min(lagBasedAutoScalerConfig.getTaskCountMax(), partitionNumbers); + if (currentActiveTaskCount == actualTaskCountMax) { log.warn("CurrentActiveTaskCount reached task count Max limit, skipping scale out action for dataSource [%s].", dataSource ); return -1; } else { - desiredActiveTaskCount = Math.min(taskCount, lagBasedAutoScalerConfig.getTaskCountMax()); + desiredActiveTaskCount = Math.min(taskCount, actualTaskCountMax); } return desiredActiveTaskCount; } - if (withinProportion >= lagBasedAutoScalerConfig.getTriggerScaleInThresholdFrequency()) { + if (withinProportion >= lagBasedAutoScalerConfig.getTriggerScaleInFractionThreshold()) { // Do Scale in int taskCount = currentActiveTaskCount - lagBasedAutoScalerConfig.getScaleInStep(); if (currentActiveTaskCount == lagBasedAutoScalerConfig.getTaskCountMin()) { diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScalerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScalerConfig.java index 26516618450e..7fa6c0030a69 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScalerConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScalerConfig.java @@ -36,8 +36,8 @@ public class LagBasedAutoScalerConfig implements AutoScalerConfig private final long scaleActionPeriodMillis; private final long scaleOutThreshold; private final long scaleInThreshold; - private final double triggerScaleOutThresholdFrequency; - private final double triggerScaleInThresholdFrequency; + private final double triggerScaleOutFractionThreshold; + private final double triggerScaleInFractionThreshold; private int taskCountMax; private int taskCountMin; private final int scaleInStep; @@ -53,8 +53,8 @@ public LagBasedAutoScalerConfig( @Nullable @JsonProperty("scaleActionPeriodMillis") Long scaleActionPeriodMillis, @Nullable @JsonProperty("scaleOutThreshold") Long scaleOutThreshold, @Nullable @JsonProperty("scaleInThreshold") Long scaleInThreshold, - @Nullable @JsonProperty("triggerScaleOutThresholdFrequency") Double triggerScaleOutThresholdFrequency, - @Nullable @JsonProperty("triggerScaleInThresholdFrequency") Double triggerScaleInThresholdFrequency, + @Nullable @JsonProperty("triggerScaleOutFractionThreshold") Double triggerScaleOutFractionThreshold, + @Nullable @JsonProperty("triggerScaleInFractionThreshold") Double triggerScaleInFractionThreshold, @JsonProperty("taskCountMax") Integer taskCountMax, @JsonProperty("taskCountMin") Integer taskCountMin, @Nullable @JsonProperty("scaleInStep") Integer scaleInStep, @@ -70,8 +70,8 @@ public LagBasedAutoScalerConfig( this.scaleActionPeriodMillis = scaleActionPeriodMillis != null ? scaleActionPeriodMillis : 60000; this.scaleOutThreshold = scaleOutThreshold != null ? scaleOutThreshold : 6000000; this.scaleInThreshold = scaleInThreshold != null ? scaleInThreshold : 1000000; - this.triggerScaleOutThresholdFrequency = triggerScaleOutThresholdFrequency != null ? triggerScaleOutThresholdFrequency : 0.3; - this.triggerScaleInThresholdFrequency = triggerScaleInThresholdFrequency != null ? triggerScaleInThresholdFrequency : 0.9; + this.triggerScaleOutFractionThreshold = triggerScaleOutFractionThreshold != null ? triggerScaleOutFractionThreshold : 0.3; + this.triggerScaleInFractionThreshold = triggerScaleInFractionThreshold != null ? triggerScaleInFractionThreshold : 0.9; // Only do taskCountMax and taskCountMin check when autoscaler is enabled. So that users left autoConfig empty{} will not throw any exception and autoscaler is disabled. // If autoscaler is disabled, no matter what configs are set, they are not used. @@ -128,15 +128,15 @@ public long getScaleInThreshold() } @JsonProperty - public double getTriggerScaleOutThresholdFrequency() + public double getTriggerScaleOutFractionThreshold() { - return triggerScaleOutThresholdFrequency; + return triggerScaleOutFractionThreshold; } @JsonProperty - public double getTriggerScaleInThresholdFrequency() + public double getTriggerScaleInFractionThreshold() { - return triggerScaleInThresholdFrequency; + return triggerScaleInFractionThreshold; } @Override diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 5c302a7d7b3a..1c4f3ec7f371 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -330,6 +330,13 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() private class TestSeekableStreamSupervisor extends BaseTestSeekableStreamSupervisor { + private int partitionNumbers; + + public TestSeekableStreamSupervisor(int partitionNumbers) + { + this.partitionNumbers = partitionNumbers; + } + @Override protected void scheduleReporting(ScheduledExecutorService reportingExec) { @@ -341,6 +348,12 @@ public LagStats computeLagStats() { return new LagStats(0, 0, 0); } + + @Override + public int getPartitionNumbers() + { + return partitionNumbers; + } } @@ -556,9 +569,9 @@ public void testAutoScalerCreated() autoScalerConfig.put("lagCollectionIntervalMillis", 500); autoScalerConfig.put("lagCollectionRangeMillis", 500); autoScalerConfig.put("scaleOutThreshold", 5000000); - autoScalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); + autoScalerConfig.put("triggerScaleOutFractionThreshold", 0.3); autoScalerConfig.put("scaleInThreshold", 1000000); - autoScalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoScalerConfig.put("triggerScaleInFractionThreshold", 0.8); autoScalerConfig.put("scaleActionStartDelayMillis", 0); autoScalerConfig.put("scaleActionPeriodMillis", 100); autoScalerConfig.put("taskCountMax", 8); @@ -686,11 +699,52 @@ public void testSeekableStreamSupervisorSpecWithScaleOut() throws InterruptedExc EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes(); EasyMock.replay(taskMaster); - TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); + TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(3); LagStats lagStats = supervisor.computeLagStats(); - LagBasedAutoScaler autoScaler = new LagBasedAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(), LagBasedAutoScalerConfig.class), spec); + LagBasedAutoScaler autoScaler = new LagBasedAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(2), LagBasedAutoScalerConfig.class), spec); + supervisor.start(); + autoScaler.start(); + supervisor.runInternal(); + int taskCountBeforeScaleOut = supervisor.getIoConfig().getTaskCount(); + Assert.assertEquals(1, taskCountBeforeScaleOut); + Thread.sleep(1 * 1000); + int taskCountAfterScaleOut = supervisor.getIoConfig().getTaskCount(); + Assert.assertEquals(2, taskCountAfterScaleOut); + + autoScaler.reset(); + autoScaler.stop(); + + } + + @Test + public void testSeekableStreamSupervisorSpecWithScaleOutSmallPartitionNumber() throws InterruptedException + { + + EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes(); + + EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes(); + EasyMock.expect(spec.getIoConfig()).andReturn(getIOConfig(1, true)).anyTimes(); + EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes(); + EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes(); + EasyMock.expect(spec.isSuspended()).andReturn(false).anyTimes(); + EasyMock.replay(spec); + + EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes(); + EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes(); + EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes(); + EasyMock.replay(ingestionSchema); + + EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes(); + EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes(); + EasyMock.replay(taskMaster); + + TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(2); + + LagStats lagStats = supervisor.computeLagStats(); + + LagBasedAutoScaler autoScaler = new LagBasedAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleOutProperties(3), LagBasedAutoScalerConfig.class), spec); supervisor.start(); autoScaler.start(); supervisor.runInternal(); @@ -727,7 +781,7 @@ public void testSeekableStreamSupervisorSpecWithScaleIn() throws InterruptedExce EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes(); EasyMock.replay(taskMaster); - TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); + TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(3); LagBasedAutoScaler autoScaler = new LagBasedAutoScaler(supervisor, DATASOURCE, mapper.convertValue(getScaleInProperties(), LagBasedAutoScalerConfig.class), spec); // enable autoscaler so that taskcount config will be ignored and init value of taskCount will use taskCountMin. @@ -781,7 +835,7 @@ public void testSeekableStreamSupervisorSpecWithScaleDisable() throws Interrupte EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes(); EasyMock.replay(taskMaster); - TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(); + TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(3); NoopTaskAutoScaler autoScaler = new NoopTaskAutoScaler(); supervisor.start(); autoScaler.start(); @@ -834,7 +888,7 @@ private SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boolean scal false, new Period("PT30M"), null, - null, mapper.convertValue(getScaleOutProperties(), AutoScalerConfig.class), null + null, mapper.convertValue(getScaleOutProperties(2), AutoScalerConfig.class), null ) {}; } else { return new SeekableStreamSupervisorIOConfig( @@ -853,19 +907,19 @@ private SeekableStreamSupervisorIOConfig getIOConfig(int taskCount, boolean scal } } - private static Map getScaleOutProperties() + private static Map getScaleOutProperties(int maxTaskCount) { HashMap autoScalerConfig = new HashMap<>(); autoScalerConfig.put("enableTaskAutoScaler", true); autoScalerConfig.put("lagCollectionIntervalMillis", 500); autoScalerConfig.put("lagCollectionRangeMillis", 500); autoScalerConfig.put("scaleOutThreshold", 0); - autoScalerConfig.put("triggerScaleOutThresholdFrequency", 0.0); + autoScalerConfig.put("triggerScaleOutFractionThreshold", 0.0); autoScalerConfig.put("scaleInThreshold", 1000000); - autoScalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoScalerConfig.put("triggerScaleInFractionThreshold", 0.8); autoScalerConfig.put("scaleActionStartDelayMillis", 0); autoScalerConfig.put("scaleActionPeriodMillis", 100); - autoScalerConfig.put("taskCountMax", 2); + autoScalerConfig.put("taskCountMax", maxTaskCount); autoScalerConfig.put("taskCountMin", 1); autoScalerConfig.put("scaleInStep", 1); autoScalerConfig.put("scaleOutStep", 2); @@ -880,9 +934,9 @@ private static Map getScaleInProperties() autoScalerConfig.put("lagCollectionIntervalMillis", 500); autoScalerConfig.put("lagCollectionRangeMillis", 500); autoScalerConfig.put("scaleOutThreshold", 8000000); - autoScalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); + autoScalerConfig.put("triggerScaleOutFractionThreshold", 0.3); autoScalerConfig.put("scaleInThreshold", 0); - autoScalerConfig.put("triggerScaleInThresholdFrequency", 0.0); + autoScalerConfig.put("triggerScaleInFractionThreshold", 0.0); autoScalerConfig.put("scaleActionStartDelayMillis", 0); autoScalerConfig.put("scaleActionPeriodMillis", 100); autoScalerConfig.put("taskCountMax", 2); diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java index 62b20a0eb9e6..42e8c557705b 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisorStateTest.java @@ -843,9 +843,9 @@ private static Map getProperties() autoScalerConfig.put("lagCollectionIntervalMillis", 500); autoScalerConfig.put("lagCollectionRangeMillis", 500); autoScalerConfig.put("scaleOutThreshold", 5000000); - autoScalerConfig.put("triggerScaleOutThresholdFrequency", 0.3); + autoScalerConfig.put("triggerScaleOutFractionThreshold", 0.3); autoScalerConfig.put("scaleInThreshold", 1000000); - autoScalerConfig.put("triggerScaleInThresholdFrequency", 0.8); + autoScalerConfig.put("triggerScaleInFractionThreshold", 0.8); autoScalerConfig.put("scaleActionStartDelayMillis", 0); autoScalerConfig.put("scaleActionPeriodMillis", 100); autoScalerConfig.put("taskCountMax", 8); diff --git a/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json b/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json index ee32c9d22fc5..f2fa82831af9 100644 --- a/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json +++ b/integration-tests/src/test/resources/stream/data/supervisor_with_autoscaler_spec_template.json @@ -53,9 +53,9 @@ "lagCollectionIntervalMillis": 500, "lagCollectionRangeMillis": 500, "scaleOutThreshold": 0, - "triggerScaleOutThresholdFrequency": 0.0, + "triggerScaleOutFractionThreshold": 0.0, "scaleInThreshold": 1000000, - "triggerScaleInThresholdFrequency": 0.9, + "triggerScaleInFractionThreshold": 0.9, "scaleActionStartDelayMillis": 0, "scaleActionPeriodMillis": 100, "taskCountMax": 2, From 644e7320ce2f51521b470b022f3406395054bd60 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Tue, 2 Mar 2021 15:33:18 +0800 Subject: [PATCH 46/47] modify docs --- docs/development/extensions-core/kafka-ingestion.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development/extensions-core/kafka-ingestion.md b/docs/development/extensions-core/kafka-ingestion.md index 88a1cf57ed95..0dd31677af9e 100644 --- a/docs/development/extensions-core/kafka-ingestion.md +++ b/docs/development/extensions-core/kafka-ingestion.md @@ -158,7 +158,7 @@ A sample supervisor spec is shown below: | `taskCountMax` | Maximum value of task count. Make Sure `taskCountMax >= taskCountMin`. If `taskCountMax > {numKafkaPartitions}`, the maximum number of reading tasks would be equal to `{numKafkaPartitions}` and `taskCountMax` would be ignored. | yes | | `taskCountMin` | Minimum value of task count. When enable autoscaler, the value of taskCount in `IOConfig` will be ignored, and `taskCountMin` will be the number of tasks that ingestion starts going up to `taskCountMax`| yes | | `minTriggerScaleActionFrequencyMillis` | Minimum time interval between two scale actions | no (default == 600000) | -| `autoScalerStrategy` | The algorithm of `autoScaler`. ONLY `lagBased` is supported for now. See [Lag Based AutoScaler Strategy Related Properties](#LagBased AutoScalerStrategy Related Properties) for details.| no (default == `lagBased`) | +| `autoScalerStrategy` | The algorithm of `autoScaler`. ONLY `lagBased` is supported for now. See [Lag Based AutoScaler Strategy Related Properties](#Lag Based AutoScaler Strategy Related Properties) for details.| no (default == `lagBased`) | ### Lag Based AutoScaler Strategy Related Properties | Property | Description | Required | From 1a9a09d05411476a27de69b0e22e8ddebe0d5f90 Mon Sep 17 00:00:00 2001 From: yuezhang Date: Fri, 5 Mar 2021 10:40:11 +0800 Subject: [PATCH 47/47] code review --- .../supervisor/KafkaSupervisorIOConfig.java | 1 + .../kinesis/supervisor/KinesisSupervisor.java | 2 +- .../supervisor/SeekableStreamSupervisor.java | 2 +- .../autoscaler/LagBasedAutoScaler.java | 6 +++--- .../autoscaler/LagBasedAutoScalerConfig.java | 21 +++++++++++++++++++ .../SeekableStreamSupervisorSpecTest.java | 2 +- 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java index 6d6a0c3f9750..87b689e6e64a 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/indexing/kafka/supervisor/KafkaSupervisorIOConfig.java @@ -121,6 +121,7 @@ public String toString() ", taskCount=" + getTaskCount() + ", taskDuration=" + getTaskDuration() + ", consumerProperties=" + consumerProperties + + ", autoScalerConfig=" + getAutoscalerConfig() + ", pollTimeout=" + pollTimeout + ", startDelay=" + getStartDelay() + ", period=" + getPeriod() + diff --git a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java index 6d6bf99fe81e..92defd231509 100644 --- a/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java +++ b/extensions-core/kinesis-indexing-service/src/main/java/org/apache/druid/indexing/kinesis/supervisor/KinesisSupervisor.java @@ -383,7 +383,7 @@ protected boolean useExclusiveStartSequenceNumberForNonFirstSequence() @Override public LagStats computeLagStats() { - return null; + throw new UnsupportedOperationException("Compute Lag Stats is not supported in KinesisSupervisor yet."); } @Override diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java index 468363dcbe84..11339d4f4ae0 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java @@ -2058,7 +2058,7 @@ protected boolean supportsPartitionExpiration() return false; } - public int getPartitionNumbers() + public int getPartitionCount() { return recordSupplier.getPartitionIds(ioConfig.getStream()).size(); } diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java index a0fe1af924dd..8c645278d55b 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScaler.java @@ -203,13 +203,13 @@ private int computeDesiredTaskCount(List lags) // Do Scale out int taskCount = currentActiveTaskCount + lagBasedAutoScalerConfig.getScaleOutStep(); - int partitionNumbers = supervisor.getPartitionNumbers(); - if (partitionNumbers <= 0) { + int partitionCount = supervisor.getPartitionCount(); + if (partitionCount <= 0) { log.warn("Partition number for [%s] <= 0 ? how can it be?", dataSource); return -1; } - int actualTaskCountMax = Math.min(lagBasedAutoScalerConfig.getTaskCountMax(), partitionNumbers); + int actualTaskCountMax = Math.min(lagBasedAutoScalerConfig.getTaskCountMax(), partitionCount); if (currentActiveTaskCount == actualTaskCountMax) { log.warn("CurrentActiveTaskCount reached task count Max limit, skipping scale out action for dataSource [%s].", dataSource diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScalerConfig.java b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScalerConfig.java index 7fa6c0030a69..0a4eb0b585e6 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScalerConfig.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/autoscaler/LagBasedAutoScalerConfig.java @@ -184,4 +184,25 @@ public long getMinTriggerScaleActionFrequencyMillis() { return minTriggerScaleActionFrequencyMillis; } + + @Override + public String toString() + { + return "autoScalerConfig{" + + "enableTaskAutoScaler=" + enableTaskAutoScaler + + ", taskCountMax=" + taskCountMax + + ", taskCountMin=" + taskCountMin + + ", minTriggerScaleActionFrequencyMillis=" + minTriggerScaleActionFrequencyMillis + + ", lagCollectionIntervalMillis=" + lagCollectionIntervalMillis + + ", lagCollectionIntervalMillis=" + lagCollectionIntervalMillis + + ", scaleOutThreshold=" + scaleOutThreshold + + ", triggerScaleOutFractionThreshold=" + triggerScaleOutFractionThreshold + + ", scaleInThreshold=" + scaleInThreshold + + ", triggerScaleInFractionThreshold=" + triggerScaleInFractionThreshold + + ", scaleActionStartDelayMillis=" + scaleActionStartDelayMillis + + ", scaleActionPeriodMillis=" + scaleActionPeriodMillis + + ", scaleInStep=" + scaleInStep + + ", scaleOutStep=" + scaleOutStep + + '}'; + } } diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java index 1c4f3ec7f371..51a39fc25525 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamSupervisorSpecTest.java @@ -350,7 +350,7 @@ public LagStats computeLagStats() } @Override - public int getPartitionNumbers() + public int getPartitionCount() { return partitionNumbers; }