From 59b6c5dcbf1982dc12cff6ee1d208d75664ee0b5 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 11 Oct 2017 12:06:55 -0500 Subject: [PATCH 1/9] Revert "1-based counts in ZkCoordinator (#4917)" This reverts commit e725ff4146e84a3f3c61b8bea10ed23d7a9e9f9b. --- .../main/java/io/druid/server/coordination/ZkCoordinator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java index 5747cd46fc9e..0d6fc933f9a7 100644 --- a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java +++ b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java @@ -261,7 +261,7 @@ public void loadLocalCache() int ignored = 0; for (int i = 0; i < segmentsToLoad.length; i++) { File file = segmentsToLoad[i]; - log.info("Loading segment cache file [%d/%d][%s].", i + 1, segmentsToLoad.length, file); + log.info("Loading segment cache file [%d/%d][%s].", i, segmentsToLoad.length, file); try { final DataSegment segment = jsonMapper.readValue(file, DataSegment.class); @@ -407,7 +407,7 @@ public void run() try { log.info( "Loading segment[%d/%d][%s]", - counter.incrementAndGet(), + counter.getAndIncrement(), numSegments, segment.getIdentifier() ); From edb9bfa84e7940783d915dfb44c4a35e03bb7ba8 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 27 Sep 2017 12:39:33 -0500 Subject: [PATCH 2/9] Revert "Remove SegmentLoaderConfig.numLoadingThreads config (#4829)" This reverts commit d8b3bfa63c9a8ff680a7eb967bc12c845cbadcd8. --- docs/content/configuration/historical.md | 3 ++- .../druid/segment/loading/SegmentLoaderConfig.java | 12 +++++++++--- .../io/druid/server/coordination/ZkCoordinator.java | 2 +- .../druid/server/coordination/ZkCoordinatorTest.java | 12 ++++++++++++ 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/content/configuration/historical.md b/docs/content/configuration/historical.md index 8ddb30c12590..27e74d1bc3f8 100644 --- a/docs/content/configuration/historical.md +++ b/docs/content/configuration/historical.md @@ -36,7 +36,8 @@ The historical node uses several of the global configs in [Configuration](../con |`druid.segmentCache.dropSegmentDelayMillis`|How long a node delays before completely dropping segment.|30000 (30 seconds)| |`druid.segmentCache.infoDir`|Historical nodes keep track of the segments they are serving so that when the process is restarted they can reload the same segments without waiting for the Coordinator to reassign. This path defines where this metadata is kept. Directory will be created if needed.|${first_location}/info_dir| |`druid.segmentCache.announceIntervalMillis`|How frequently to announce segments while segments are loading from cache. Set this value to zero to wait for all segments to be loaded before announcing.|5000 (5 seconds)| -|`druid.segmentCache.numBootstrapThreads`|How many segments to load concurrently from local storage at startup.|1| +|`druid.segmentCache.numLoadingThreads`|How many segments to load concurrently from from deep storage.|1| +|`druid.segmentCache.numBootstrapThreads`|How many segments to load concurrently from local storage at startup.|Same as numLoadingThreads| ### Query Configs diff --git a/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java b/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java index a0d66c1f402e..f67cd649bfa0 100644 --- a/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java +++ b/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java @@ -31,8 +31,6 @@ */ public class SegmentLoaderConfig { - private static final int DEFAULT_NUM_BOOTSTRAP_THREADS = 1; - @JsonProperty @NotEmpty private List locations = null; @@ -46,6 +44,9 @@ public class SegmentLoaderConfig @JsonProperty("announceIntervalMillis") private int announceIntervalMillis = 0; // do not background announce + @JsonProperty("numLoadingThreads") + private int numLoadingThreads = 1; + @JsonProperty("numBootstrapThreads") private Integer numBootstrapThreads = null; @@ -72,9 +73,14 @@ public int getAnnounceIntervalMillis() return announceIntervalMillis; } + public int getNumLoadingThreads() + { + return numLoadingThreads; + } + public int getNumBootstrapThreads() { - return numBootstrapThreads == null ? DEFAULT_NUM_BOOTSTRAP_THREADS : numBootstrapThreads; + return numBootstrapThreads == null ? numLoadingThreads : numBootstrapThreads; } public File getInfoDir() diff --git a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java index 0d6fc933f9a7..4ae8887694eb 100644 --- a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java +++ b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java @@ -127,7 +127,7 @@ public void start() throws IOException loadQueueLocation, true, true, - Execs.singleThreaded("ZkCoordinator-%s") + Execs.multiThreaded(config.getNumLoadingThreads(), "ZkCoordinator-%s") ); try { diff --git a/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java index fbd407265280..dc14592b820b 100644 --- a/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java +++ b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java @@ -202,6 +202,12 @@ public File getInfoDir() return infoDir; } + @Override + public int getNumLoadingThreads() + { + return 5; + } + @Override public int getAnnounceIntervalMillis() { @@ -487,6 +493,12 @@ public File getInfoDir() return infoDir; } + @Override + public int getNumLoadingThreads() + { + return 5; + } + @Override public int getAnnounceIntervalMillis() { From 6631ead59cf24450e387a6ca1a30da9189513bc4 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Fri, 8 Sep 2017 11:38:24 -0500 Subject: [PATCH 3/9] introducing CuratorLoadQueuePeon --- .../coordinator/CuratorLoadQueuePeon.java | 509 ++++++++++++++++++ .../server/coordinator/LoadQueuePeon.java | 471 +--------------- .../coordinator/LoadQueueTaskMaster.java | 2 +- 3 files changed, 523 insertions(+), 459 deletions(-) create mode 100644 server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java diff --git a/server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java b/server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java new file mode 100644 index 000000000000..8ab64a466d3c --- /dev/null +++ b/server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java @@ -0,0 +1,509 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.coordinator; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import com.metamx.emitter.EmittingLogger; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.concurrent.ScheduledExecutors; +import io.druid.server.coordination.DataSegmentChangeRequest; +import io.druid.server.coordination.SegmentChangeRequestDrop; +import io.druid.server.coordination.SegmentChangeRequestLoad; +import io.druid.server.coordination.SegmentChangeRequestNoop; +import io.druid.timeline.DataSegment; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.CuratorWatcher; +import org.apache.curator.utils.ZKPaths; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.data.Stat; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + */ +public class CuratorLoadQueuePeon extends LoadQueuePeon +{ + private static final EmittingLogger log = new EmittingLogger(CuratorLoadQueuePeon.class); + private static final int DROP = 0; + private static final int LOAD = 1; + + private static void executeCallbacks(List callbacks) + { + for (LoadPeonCallback callback : callbacks) { + if (callback != null) { + callback.execute(); + } + } + } + + private final CuratorFramework curator; + private final String basePath; + private final ObjectMapper jsonMapper; + private final ScheduledExecutorService processingExecutor; + private final ExecutorService callBackExecutor; + private final DruidCoordinatorConfig config; + + private final AtomicLong queuedSize = new AtomicLong(0); + private final AtomicInteger failedAssignCount = new AtomicInteger(0); + + private final ConcurrentSkipListMap segmentsToLoad = new ConcurrentSkipListMap<>( + DruidCoordinator.SEGMENT_COMPARATOR + ); + private final ConcurrentSkipListMap segmentsToDrop = new ConcurrentSkipListMap<>( + DruidCoordinator.SEGMENT_COMPARATOR + ); + private final ConcurrentSkipListSet segmentsMarkedToDrop = new ConcurrentSkipListSet<>( + DruidCoordinator.SEGMENT_COMPARATOR + ); + + private final Object lock = new Object(); + + private volatile SegmentHolder currentlyProcessing = null; + private boolean stopped = false; + + CuratorLoadQueuePeon( + CuratorFramework curator, + String basePath, + ObjectMapper jsonMapper, + ScheduledExecutorService processingExecutor, + ExecutorService callbackExecutor, + DruidCoordinatorConfig config + ) + { + this.curator = curator; + this.basePath = basePath; + this.jsonMapper = jsonMapper; + this.callBackExecutor = callbackExecutor; + this.processingExecutor = processingExecutor; + this.config = config; + } + + @JsonProperty + @Override + public Set getSegmentsToLoad() + { + return segmentsToLoad.keySet(); + } + + @JsonProperty + @Override + public Set getSegmentsToDrop() + { + return segmentsToDrop.keySet(); + } + + @JsonProperty + @Override + public Set getSegmentsMarkedToDrop() + { + return segmentsMarkedToDrop; + } + + @Override + public long getLoadQueueSize() + { + return queuedSize.get(); + } + + @Override + public int getAndResetFailedAssignCount() + { + return failedAssignCount.getAndSet(0); + } + + @Override + public int getNumberOfSegmentsInQueue() + { + return segmentsToLoad.size(); + } + + @Override + public void loadSegment( + final DataSegment segment, + final LoadPeonCallback callback + ) + { + synchronized (lock) { + if ((currentlyProcessing != null) && + currentlyProcessing.getSegmentIdentifier().equals(segment.getIdentifier())) { + if (callback != null) { + currentlyProcessing.addCallback(callback); + } + return; + } + } + + synchronized (lock) { + final SegmentHolder existingHolder = segmentsToLoad.get(segment); + if (existingHolder != null) { + if ((callback != null)) { + existingHolder.addCallback(callback); + } + return; + } + } + + log.info("Asking server peon[%s] to load segment[%s]", basePath, segment.getIdentifier()); + queuedSize.addAndGet(segment.getSize()); + segmentsToLoad.put(segment, new SegmentHolder(segment, LOAD, Collections.singletonList(callback))); + } + + @Override + public void dropSegment( + final DataSegment segment, + final LoadPeonCallback callback + ) + { + synchronized (lock) { + if ((currentlyProcessing != null) && + currentlyProcessing.getSegmentIdentifier().equals(segment.getIdentifier())) { + if (callback != null) { + currentlyProcessing.addCallback(callback); + } + return; + } + } + + synchronized (lock) { + final SegmentHolder existingHolder = segmentsToDrop.get(segment); + if (existingHolder != null) { + if (callback != null) { + existingHolder.addCallback(callback); + } + return; + } + } + + log.info("Asking server peon[%s] to drop segment[%s]", basePath, segment.getIdentifier()); + segmentsToDrop.put(segment, new SegmentHolder(segment, DROP, Collections.singletonList(callback))); + } + + @Override + public void markSegmentToDrop(DataSegment dataSegment) + { + segmentsMarkedToDrop.add(dataSegment); + } + + @Override + public void unmarkSegmentToDrop(DataSegment dataSegment) + { + segmentsMarkedToDrop.remove(dataSegment); + } + + private void processSegmentChangeRequest() + { + if (currentlyProcessing != null) { + log.debug( + "Server[%s] skipping processSegmentChangeRequest because something is currently loading[%s].", + basePath, + currentlyProcessing.getSegmentIdentifier() + ); + + return; + } + + if (!segmentsToDrop.isEmpty()) { + currentlyProcessing = segmentsToDrop.firstEntry().getValue(); + log.info("Server[%s] dropping [%s]", basePath, currentlyProcessing.getSegmentIdentifier()); + } else if (!segmentsToLoad.isEmpty()) { + currentlyProcessing = segmentsToLoad.firstEntry().getValue(); + log.info("Server[%s] loading [%s]", basePath, currentlyProcessing.getSegmentIdentifier()); + } else { + return; + } + + try { + if (currentlyProcessing == null) { + if (!stopped) { + log.makeAlert("Crazy race condition! server[%s]", basePath) + .emit(); + } + actionCompleted(); + return; + } + + log.info("Server[%s] processing segment[%s]", basePath, currentlyProcessing.getSegmentIdentifier()); + final String path = ZKPaths.makePath(basePath, currentlyProcessing.getSegmentIdentifier()); + final byte[] payload = jsonMapper.writeValueAsBytes(currentlyProcessing.getChangeRequest()); + curator.create().withMode(CreateMode.EPHEMERAL).forPath(path, payload); + + processingExecutor.schedule( + new Runnable() + { + @Override + public void run() + { + try { + if (curator.checkExists().forPath(path) != null) { + failAssign(new ISE("%s was never removed! Failing this operation!", path)); + } + } + catch (Exception e) { + failAssign(e); + } + } + }, + config.getLoadTimeoutDelay().getMillis(), + TimeUnit.MILLISECONDS + ); + + final Stat stat = curator.checkExists().usingWatcher( + new CuratorWatcher() + { + @Override + public void process(WatchedEvent watchedEvent) throws Exception + { + switch (watchedEvent.getType()) { + case NodeDeleted: + entryRemoved(watchedEvent.getPath()); + break; + default: + // do nothing + } + } + } + ).forPath(path); + + if (stat == null) { + final byte[] noopPayload = jsonMapper.writeValueAsBytes(new SegmentChangeRequestNoop()); + + // Create a node and then delete it to remove the registered watcher. This is a work-around for + // a zookeeper race condition. Specifically, when you set a watcher, it fires on the next event + // that happens for that node. If no events happen, the watcher stays registered foreverz. + // Couple that with the fact that you cannot set a watcher when you create a node, but what we + // want is to create a node and then watch for it to get deleted. The solution is that you *can* + // set a watcher when you check to see if it exists so, we first create the node and then set a + // watcher on its existence. However, if already does not exist by the time the existence check + // returns, then the watcher that was set will never fire (nobody will ever create the node + // again) and thus lead to a slow, but real, memory leak. So, we create another node to cause + // that watcher to fire and delete it right away. + // + // We do not create the existence watcher first, because then it will fire when we create the + // node and we'll have the same race when trying to refresh that watcher. + curator.create().withMode(CreateMode.EPHEMERAL).forPath(path, noopPayload); + + entryRemoved(path); + } + } + catch (Exception e) { + failAssign(e); + } + } + + private void actionCompleted() + { + if (currentlyProcessing != null) { + switch (currentlyProcessing.getType()) { + case LOAD: + segmentsToLoad.remove(currentlyProcessing.getSegment()); + queuedSize.addAndGet(-currentlyProcessing.getSegmentSize()); + break; + case DROP: + segmentsToDrop.remove(currentlyProcessing.getSegment()); + break; + default: + throw new UnsupportedOperationException(); + } + + final List callbacks = currentlyProcessing.getCallbacks(); + currentlyProcessing = null; + callBackExecutor.execute( + new Runnable() + { + @Override + public void run() + { + executeCallbacks(callbacks); + } + } + ); + } + } + + @Override + public void start() + { + ScheduledExecutors.scheduleAtFixedRate( + processingExecutor, + config.getLoadQueuePeonRepeatDelay(), + config.getLoadQueuePeonRepeatDelay(), + new Callable() + { + @Override + public ScheduledExecutors.Signal call() + { + processSegmentChangeRequest(); + + if (stopped) { + return ScheduledExecutors.Signal.STOP; + } else { + return ScheduledExecutors.Signal.REPEAT; + } + } + } + ); + } + + @Override + public void stop() + { + synchronized (lock) { + if (currentlyProcessing != null) { + executeCallbacks(currentlyProcessing.getCallbacks()); + currentlyProcessing = null; + } + + if (!segmentsToDrop.isEmpty()) { + for (SegmentHolder holder : segmentsToDrop.values()) { + executeCallbacks(holder.getCallbacks()); + } + } + segmentsToDrop.clear(); + + if (!segmentsToLoad.isEmpty()) { + for (SegmentHolder holder : segmentsToLoad.values()) { + executeCallbacks(holder.getCallbacks()); + } + } + segmentsToLoad.clear(); + + queuedSize.set(0L); + failedAssignCount.set(0); + stopped = true; + } + } + + private void entryRemoved(String path) + { + synchronized (lock) { + if (currentlyProcessing == null) { + log.warn("Server[%s] an entry[%s] was removed even though it wasn't loading!?", basePath, path); + return; + } + if (!ZKPaths.getNodeFromPath(path).equals(currentlyProcessing.getSegmentIdentifier())) { + log.warn( + "Server[%s] entry [%s] was removed even though it's not what is currently loading[%s]", + basePath, path, currentlyProcessing + ); + return; + } + actionCompleted(); + log.info("Server[%s] done processing [%s]", basePath, path); + } + } + + private void failAssign(Exception e) + { + synchronized (lock) { + log.error(e, "Server[%s], throwable caught when submitting [%s].", basePath, currentlyProcessing); + failedAssignCount.getAndIncrement(); + // Act like it was completed so that the coordinator gives it to someone else + actionCompleted(); + } + } + + private static class SegmentHolder + { + private final DataSegment segment; + private final DataSegmentChangeRequest changeRequest; + private final int type; + private final List callbacks = Lists.newArrayList(); + + private SegmentHolder( + DataSegment segment, + int type, + Collection callbacks + ) + { + this.segment = segment; + this.type = type; + this.changeRequest = (type == LOAD) + ? new SegmentChangeRequestLoad(segment) + : new SegmentChangeRequestDrop(segment); + this.callbacks.addAll(callbacks); + } + + public DataSegment getSegment() + { + return segment; + } + + public int getType() + { + return type; + } + + public String getSegmentIdentifier() + { + return segment.getIdentifier(); + } + + public long getSegmentSize() + { + return segment.getSize(); + } + + public void addCallbacks(Collection newCallbacks) + { + synchronized (callbacks) { + callbacks.addAll(newCallbacks); + } + } + + public void addCallback(LoadPeonCallback newCallback) + { + synchronized (callbacks) { + callbacks.add(newCallback); + } + } + + public List getCallbacks() + { + synchronized (callbacks) { + return callbacks; + } + } + + public DataSegmentChangeRequest getChangeRequest() + { + return changeRequest; + } + + @Override + public String toString() + { + return changeRequest.toString(); + } + } +} diff --git a/server/src/main/java/io/druid/server/coordinator/LoadQueuePeon.java b/server/src/main/java/io/druid/server/coordinator/LoadQueuePeon.java index 81eab183bdcf..b46ff5844608 100644 --- a/server/src/main/java/io/druid/server/coordinator/LoadQueuePeon.java +++ b/server/src/main/java/io/druid/server/coordinator/LoadQueuePeon.java @@ -19,479 +19,34 @@ package io.druid.server.coordinator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import com.metamx.emitter.EmittingLogger; -import io.druid.java.util.common.ISE; -import io.druid.java.util.common.concurrent.ScheduledExecutors; -import io.druid.server.coordination.DataSegmentChangeRequest; -import io.druid.server.coordination.SegmentChangeRequestDrop; -import io.druid.server.coordination.SegmentChangeRequestLoad; -import io.druid.server.coordination.SegmentChangeRequestNoop; import io.druid.timeline.DataSegment; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.api.CuratorWatcher; -import org.apache.curator.utils.ZKPaths; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.data.Stat; -import java.util.Collection; -import java.util.Collections; -import java.util.List; import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; /** */ -public class LoadQueuePeon +public abstract class LoadQueuePeon { - private static final EmittingLogger log = new EmittingLogger(LoadQueuePeon.class); - private static final int DROP = 0; - private static final int LOAD = 1; + public abstract void start(); + public abstract void stop(); - private static void executeCallbacks(List callbacks) - { - for (LoadPeonCallback callback : callbacks) { - if (callback != null) { - callback.execute(); - } - } - } + public abstract Set getSegmentsToLoad(); - private final CuratorFramework curator; - private final String basePath; - private final ObjectMapper jsonMapper; - private final ScheduledExecutorService processingExecutor; - private final ExecutorService callBackExecutor; - private final DruidCoordinatorConfig config; + public abstract Set getSegmentsToDrop(); - private final AtomicLong queuedSize = new AtomicLong(0); - private final AtomicInteger failedAssignCount = new AtomicInteger(0); + public abstract void unmarkSegmentToDrop(DataSegment segmentToLoad); - private final ConcurrentSkipListMap segmentsToLoad = new ConcurrentSkipListMap<>( - DruidCoordinator.SEGMENT_COMPARATOR - ); - private final ConcurrentSkipListMap segmentsToDrop = new ConcurrentSkipListMap<>( - DruidCoordinator.SEGMENT_COMPARATOR - ); - private final ConcurrentSkipListSet segmentsMarkedToDrop = new ConcurrentSkipListSet<>( - DruidCoordinator.SEGMENT_COMPARATOR - ); - private final Object lock = new Object(); + public abstract void markSegmentToDrop(DataSegment segmentToLoad); - private volatile SegmentHolder currentlyProcessing = null; - private boolean stopped = false; + public abstract void loadSegment(DataSegment segment, LoadPeonCallback callback); + public abstract void dropSegment(DataSegment segment, LoadPeonCallback callback); - LoadQueuePeon( - CuratorFramework curator, - String basePath, - ObjectMapper jsonMapper, - ScheduledExecutorService processingExecutor, - ExecutorService callbackExecutor, - DruidCoordinatorConfig config - ) - { - this.curator = curator; - this.basePath = basePath; - this.jsonMapper = jsonMapper; - this.callBackExecutor = callbackExecutor; - this.processingExecutor = processingExecutor; - this.config = config; - } + public abstract long getLoadQueueSize(); - @JsonProperty - public Set getSegmentsToLoad() - { - return segmentsToLoad.keySet(); - } + public abstract int getAndResetFailedAssignCount(); - @JsonProperty - public Set getSegmentsToDrop() - { - return segmentsToDrop.keySet(); - } + public abstract int getNumberOfSegmentsInQueue(); + public abstract Set getSegmentsMarkedToDrop(); - @JsonProperty - public Set getSegmentsMarkedToDrop() - { - return segmentsMarkedToDrop; - } - - public long getLoadQueueSize() - { - return queuedSize.get(); - } - - public int getAndResetFailedAssignCount() - { - return failedAssignCount.getAndSet(0); - } - - public int getNumberOfSegmentsInQueue() - { - return segmentsToLoad.size(); - } - - public void loadSegment( - final DataSegment segment, - final LoadPeonCallback callback - ) - { - synchronized (lock) { - if ((currentlyProcessing != null) && - currentlyProcessing.getSegmentIdentifier().equals(segment.getIdentifier())) { - if (callback != null) { - currentlyProcessing.addCallback(callback); - } - return; - } - } - - synchronized (lock) { - final SegmentHolder existingHolder = segmentsToLoad.get(segment); - if (existingHolder != null) { - if ((callback != null)) { - existingHolder.addCallback(callback); - } - return; - } - } - - log.info("Asking server peon[%s] to load segment[%s]", basePath, segment.getIdentifier()); - queuedSize.addAndGet(segment.getSize()); - segmentsToLoad.put(segment, new SegmentHolder(segment, LOAD, Collections.singletonList(callback))); - } - - public void dropSegment( - final DataSegment segment, - final LoadPeonCallback callback - ) - { - synchronized (lock) { - if ((currentlyProcessing != null) && - currentlyProcessing.getSegmentIdentifier().equals(segment.getIdentifier())) { - if (callback != null) { - currentlyProcessing.addCallback(callback); - } - return; - } - } - - synchronized (lock) { - final SegmentHolder existingHolder = segmentsToDrop.get(segment); - if (existingHolder != null) { - if (callback != null) { - existingHolder.addCallback(callback); - } - return; - } - } - - log.info("Asking server peon[%s] to drop segment[%s]", basePath, segment.getIdentifier()); - segmentsToDrop.put(segment, new SegmentHolder(segment, DROP, Collections.singletonList(callback))); - } - - public void markSegmentToDrop(DataSegment dataSegment) - { - segmentsMarkedToDrop.add(dataSegment); - } - - public void unmarkSegmentToDrop(DataSegment dataSegment) - { - segmentsMarkedToDrop.remove(dataSegment); - } - - private void processSegmentChangeRequest() - { - if (currentlyProcessing != null) { - log.debug( - "Server[%s] skipping processSegmentChangeRequest because something is currently loading[%s].", - basePath, - currentlyProcessing.getSegmentIdentifier() - ); - - return; - } - - if (!segmentsToDrop.isEmpty()) { - currentlyProcessing = segmentsToDrop.firstEntry().getValue(); - log.info("Server[%s] dropping [%s]", basePath, currentlyProcessing.getSegmentIdentifier()); - } else if (!segmentsToLoad.isEmpty()) { - currentlyProcessing = segmentsToLoad.firstEntry().getValue(); - log.info("Server[%s] loading [%s]", basePath, currentlyProcessing.getSegmentIdentifier()); - } else { - return; - } - - try { - if (currentlyProcessing == null) { - if (!stopped) { - log.makeAlert("Crazy race condition! server[%s]", basePath) - .emit(); - } - actionCompleted(); - return; - } - - log.info("Server[%s] processing segment[%s]", basePath, currentlyProcessing.getSegmentIdentifier()); - final String path = ZKPaths.makePath(basePath, currentlyProcessing.getSegmentIdentifier()); - final byte[] payload = jsonMapper.writeValueAsBytes(currentlyProcessing.getChangeRequest()); - curator.create().withMode(CreateMode.EPHEMERAL).forPath(path, payload); - - processingExecutor.schedule( - new Runnable() - { - @Override - public void run() - { - try { - if (curator.checkExists().forPath(path) != null) { - failAssign(new ISE("%s was never removed! Failing this operation!", path)); - } - } - catch (Exception e) { - failAssign(e); - } - } - }, - config.getLoadTimeoutDelay().getMillis(), - TimeUnit.MILLISECONDS - ); - - final Stat stat = curator.checkExists().usingWatcher( - new CuratorWatcher() - { - @Override - public void process(WatchedEvent watchedEvent) throws Exception - { - switch (watchedEvent.getType()) { - case NodeDeleted: - entryRemoved(watchedEvent.getPath()); - break; - default: - // do nothing - } - } - } - ).forPath(path); - - if (stat == null) { - final byte[] noopPayload = jsonMapper.writeValueAsBytes(new SegmentChangeRequestNoop()); - - // Create a node and then delete it to remove the registered watcher. This is a work-around for - // a zookeeper race condition. Specifically, when you set a watcher, it fires on the next event - // that happens for that node. If no events happen, the watcher stays registered foreverz. - // Couple that with the fact that you cannot set a watcher when you create a node, but what we - // want is to create a node and then watch for it to get deleted. The solution is that you *can* - // set a watcher when you check to see if it exists so, we first create the node and then set a - // watcher on its existence. However, if already does not exist by the time the existence check - // returns, then the watcher that was set will never fire (nobody will ever create the node - // again) and thus lead to a slow, but real, memory leak. So, we create another node to cause - // that watcher to fire and delete it right away. - // - // We do not create the existence watcher first, because then it will fire when we create the - // node and we'll have the same race when trying to refresh that watcher. - curator.create().withMode(CreateMode.EPHEMERAL).forPath(path, noopPayload); - - entryRemoved(path); - } - } - catch (Exception e) { - failAssign(e); - } - } - - private void actionCompleted() - { - if (currentlyProcessing != null) { - switch (currentlyProcessing.getType()) { - case LOAD: - segmentsToLoad.remove(currentlyProcessing.getSegment()); - queuedSize.addAndGet(-currentlyProcessing.getSegmentSize()); - break; - case DROP: - segmentsToDrop.remove(currentlyProcessing.getSegment()); - break; - default: - throw new UnsupportedOperationException(); - } - - final List callbacks = currentlyProcessing.getCallbacks(); - currentlyProcessing = null; - callBackExecutor.execute( - new Runnable() - { - @Override - public void run() - { - executeCallbacks(callbacks); - } - } - ); - } - } - - public void start() - { - ScheduledExecutors.scheduleAtFixedRate( - processingExecutor, - config.getLoadQueuePeonRepeatDelay(), - config.getLoadQueuePeonRepeatDelay(), - new Callable() - { - @Override - public ScheduledExecutors.Signal call() - { - processSegmentChangeRequest(); - - if (stopped) { - return ScheduledExecutors.Signal.STOP; - } else { - return ScheduledExecutors.Signal.REPEAT; - } - } - } - ); - } - - public void stop() - { - synchronized (lock) { - if (currentlyProcessing != null) { - executeCallbacks(currentlyProcessing.getCallbacks()); - currentlyProcessing = null; - } - - if (!segmentsToDrop.isEmpty()) { - for (SegmentHolder holder : segmentsToDrop.values()) { - executeCallbacks(holder.getCallbacks()); - } - } - segmentsToDrop.clear(); - - if (!segmentsToLoad.isEmpty()) { - for (SegmentHolder holder : segmentsToLoad.values()) { - executeCallbacks(holder.getCallbacks()); - } - } - segmentsToLoad.clear(); - - queuedSize.set(0L); - failedAssignCount.set(0); - stopped = true; - } - } - - private void entryRemoved(String path) - { - synchronized (lock) { - if (currentlyProcessing == null) { - log.warn("Server[%s] an entry[%s] was removed even though it wasn't loading!?", basePath, path); - return; - } - if (!ZKPaths.getNodeFromPath(path).equals(currentlyProcessing.getSegmentIdentifier())) { - log.warn( - "Server[%s] entry [%s] was removed even though it's not what is currently loading[%s]", - basePath, path, currentlyProcessing - ); - return; - } - actionCompleted(); - log.info("Server[%s] done processing [%s]", basePath, path); - } - } - - private void failAssign(Exception e) - { - synchronized (lock) { - log.error(e, "Server[%s], throwable caught when submitting [%s].", basePath, currentlyProcessing); - failedAssignCount.getAndIncrement(); - // Act like it was completed so that the coordinator gives it to someone else - actionCompleted(); - } - } - - private static class SegmentHolder - { - private final DataSegment segment; - private final DataSegmentChangeRequest changeRequest; - private final int type; - private final List callbacks = Lists.newArrayList(); - - private SegmentHolder( - DataSegment segment, - int type, - Collection callbacks - ) - { - this.segment = segment; - this.type = type; - this.changeRequest = (type == LOAD) - ? new SegmentChangeRequestLoad(segment) - : new SegmentChangeRequestDrop(segment); - this.callbacks.addAll(callbacks); - } - - public DataSegment getSegment() - { - return segment; - } - - public int getType() - { - return type; - } - - public String getSegmentIdentifier() - { - return segment.getIdentifier(); - } - - public long getSegmentSize() - { - return segment.getSize(); - } - - public void addCallbacks(Collection newCallbacks) - { - synchronized (callbacks) { - callbacks.addAll(newCallbacks); - } - } - - public void addCallback(LoadPeonCallback newCallback) - { - synchronized (callbacks) { - callbacks.add(newCallback); - } - } - - public List getCallbacks() - { - synchronized (callbacks) { - return callbacks; - } - } - - public DataSegmentChangeRequest getChangeRequest() - { - return changeRequest; - } - - @Override - public String toString() - { - return changeRequest.toString(); - } - } } diff --git a/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java b/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java index cdad8c7adc21..367198060735 100644 --- a/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java +++ b/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java @@ -55,6 +55,6 @@ public LoadQueueTaskMaster( public LoadQueuePeon giveMePeon(String basePath) { - return new LoadQueuePeon(curator, basePath, jsonMapper, peonExec, callbackExec, config); + return new CuratorLoadQueuePeon(curator, basePath, jsonMapper, peonExec, callbackExec, config); } } From edcd0a2d32f05010cff7b677c94df41c7294855a Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Fri, 8 Sep 2017 11:42:12 -0500 Subject: [PATCH 4/9] HttpLoadQueuePeon based off of current code --- .../server/coordinator/HttpLoadQueuePeon.java | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java diff --git a/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java b/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java new file mode 100644 index 000000000000..be7466003576 --- /dev/null +++ b/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java @@ -0,0 +1,208 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.coordinator; + +import com.google.common.collect.Lists; +import io.druid.server.coordination.DataSegmentChangeRequest; +import io.druid.server.coordination.SegmentChangeRequestDrop; +import io.druid.server.coordination.SegmentChangeRequestLoad; +import io.druid.timeline.DataSegment; + +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + */ +public class HttpLoadQueuePeon extends LoadQueuePeon +{ + private static final int DROP = 0; + private static final int LOAD = 1; + + private final AtomicLong queuedSize = new AtomicLong(0); + private final AtomicInteger failedAssignCount = new AtomicInteger(0); + + private final ConcurrentSkipListMap segmentsToLoad = new ConcurrentSkipListMap<>( + DruidCoordinator.SEGMENT_COMPARATOR + ); + private final ConcurrentSkipListMap segmentsToDrop = new ConcurrentSkipListMap<>( + DruidCoordinator.SEGMENT_COMPARATOR + ); + private final ConcurrentSkipListSet segmentsMarkedToDrop = new ConcurrentSkipListSet<>( + DruidCoordinator.SEGMENT_COMPARATOR + ); + + private void mainLoop() + { + //check status of each pending and update + + //check size of pending list and send request for (MAX - pendingList.size) + } + + @Override + public void start() + { + + } + + @Override + public void stop() + { + + } + + @Override + public Set getSegmentsToLoad() + { + return segmentsToLoad.keySet(); + } + + @Override + public Set getSegmentsToDrop() + { + return segmentsToDrop.keySet(); + } + + @Override + public long getLoadQueueSize() + { + return queuedSize.get(); + } + + @Override + public int getAndResetFailedAssignCount() + { + return failedAssignCount.getAndSet(0); + } + + @Override + public void markSegmentToDrop(DataSegment dataSegment) + { + segmentsMarkedToDrop.add(dataSegment); + } + + @Override + public void unmarkSegmentToDrop(DataSegment dataSegment) + { + segmentsMarkedToDrop.remove(dataSegment); + } + + @Override + public int getNumberOfSegmentsInQueue() + { + return segmentsToLoad.size(); + } + + @Override + public Set getSegmentsMarkedToDrop() + { + return segmentsMarkedToDrop; + } + + @Override + public void loadSegment(DataSegment segment, LoadPeonCallback callback) + { + + } + + @Override + public void dropSegment(DataSegment segment, LoadPeonCallback callback) + { + + } + + private static class SegmentHolder + { + private final DataSegment segment; + private final DataSegmentChangeRequest changeRequest; + private final int type; + private final List callbacks = Lists.newArrayList(); + + private SegmentHolder( + DataSegment segment, + int type, + Collection callbacks + ) + { + this.segment = segment; + this.type = type; + this.changeRequest = (type == LOAD) + ? new SegmentChangeRequestLoad(segment) + : new SegmentChangeRequestDrop(segment); + this.callbacks.addAll(callbacks); + } + + public DataSegment getSegment() + { + return segment; + } + + public int getType() + { + return type; + } + + public String getSegmentIdentifier() + { + return segment.getIdentifier(); + } + + public long getSegmentSize() + { + return segment.getSize(); + } + + public void addCallbacks(Collection newCallbacks) + { + synchronized (callbacks) { + callbacks.addAll(newCallbacks); + } + } + + public void addCallback(LoadPeonCallback newCallback) + { + synchronized (callbacks) { + callbacks.add(newCallback); + } + } + + public List getCallbacks() + { + synchronized (callbacks) { + return callbacks; + } + } + + public DataSegmentChangeRequest getChangeRequest() + { + return changeRequest; + } + + @Override + public String toString() + { + return changeRequest.toString(); + } + } +} From 92f88519ee1356b3a730434d647d5c88a5a4c528 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 27 Sep 2017 13:13:04 -0500 Subject: [PATCH 5/9] SegmentLoadDropHandler copy/pasted from ZkCoordinator --- .../coordination/SegmentLoadDropHandler.java | 645 ++++++++++++++++++ 1 file changed, 645 insertions(+) create mode 100644 server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java diff --git a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java new file mode 100644 index 000000000000..ed787f692add --- /dev/null +++ b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java @@ -0,0 +1,645 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.coordination; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Queues; +import com.google.common.util.concurrent.SettableFuture; +import com.google.inject.Inject; +import com.metamx.emitter.EmittingLogger; +import io.druid.concurrent.Execs; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.concurrent.ScheduledExecutorFactory; +import io.druid.java.util.common.lifecycle.LifecycleStart; +import io.druid.java.util.common.lifecycle.LifecycleStop; +import io.druid.segment.loading.SegmentLoaderConfig; +import io.druid.segment.loading.SegmentLoadingException; +import io.druid.server.SegmentManager; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.PathChildrenCache; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import org.apache.curator.utils.ZKPaths; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + */ +public class ZkCoordinator implements DataSegmentChangeHandler +{ + private static final EmittingLogger log = new EmittingLogger(ZkCoordinator.class); + + private final Object lock = new Object(); + + private final ObjectMapper jsonMapper; + private final ZkPathsConfig zkPaths; + private final SegmentLoaderConfig config; + private final DruidServerMetadata me; + private final CuratorFramework curator; + private final DataSegmentAnnouncer announcer; + private final DataSegmentServerAnnouncer serverAnnouncer; + private final SegmentManager segmentManager; + private final ScheduledExecutorService exec; + private final ConcurrentSkipListSet segmentsToDelete; + + + private volatile PathChildrenCache loadQueueCache; + private volatile boolean started = false; + + @Inject + public ZkCoordinator( + ObjectMapper jsonMapper, + SegmentLoaderConfig config, + ZkPathsConfig zkPaths, + DruidServerMetadata me, + DataSegmentAnnouncer announcer, + DataSegmentServerAnnouncer serverAnnouncer, + CuratorFramework curator, + SegmentManager segmentManager, + ScheduledExecutorFactory factory + ) + { + this.jsonMapper = jsonMapper; + this.zkPaths = zkPaths; + this.config = config; + this.me = me; + this.curator = curator; + this.announcer = announcer; + this.serverAnnouncer = serverAnnouncer; + this.segmentManager = segmentManager; + + this.exec = factory.create(1, "ZkCoordinator-Exec--%d"); + this.segmentsToDelete = new ConcurrentSkipListSet<>(); + } + + @LifecycleStart + public void start() throws IOException + { + synchronized (lock) { + if (started) { + return; + } + + log.info("Starting zkCoordinator for server[%s]", me.getName()); + + final String loadQueueLocation = ZKPaths.makePath(zkPaths.getLoadQueuePath(), me.getName()); + final String servedSegmentsLocation = ZKPaths.makePath(zkPaths.getServedSegmentsPath(), me.getName()); + final String liveSegmentsLocation = ZKPaths.makePath(zkPaths.getLiveSegmentsPath(), me.getName()); + + loadQueueCache = new PathChildrenCache( + curator, + loadQueueLocation, + true, + true, + Execs.multiThreaded(config.getNumLoadingThreads(), "ZkCoordinator-%s") + ); + + try { + curator.newNamespaceAwareEnsurePath(loadQueueLocation).ensure(curator.getZookeeperClient()); + curator.newNamespaceAwareEnsurePath(servedSegmentsLocation).ensure(curator.getZookeeperClient()); + curator.newNamespaceAwareEnsurePath(liveSegmentsLocation).ensure(curator.getZookeeperClient()); + + loadLocalCache(); + serverAnnouncer.announce(); + + loadQueueCache.getListenable().addListener( + new PathChildrenCacheListener() + { + @Override + public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception + { + final ChildData child = event.getData(); + switch (event.getType()) { + case CHILD_ADDED: + final String path = child.getPath(); + final DataSegmentChangeRequest request = jsonMapper.readValue( + child.getData(), DataSegmentChangeRequest.class + ); + + log.info("New request[%s] with zNode[%s].", request.asString(), path); + + try { + request.go( + getDataSegmentChangeHandler(), + new DataSegmentChangeCallback() + { + boolean hasRun = false; + + @Override + public void execute() + { + try { + if (!hasRun) { + curator.delete().guaranteed().forPath(path); + log.info("Completed request [%s]", request.asString()); + hasRun = true; + } + } + catch (Exception e) { + try { + curator.delete().guaranteed().forPath(path); + } + catch (Exception e1) { + log.error(e1, "Failed to delete zNode[%s], but ignoring exception.", path); + } + log.error(e, "Exception while removing zNode[%s]", path); + throw Throwables.propagate(e); + } + } + } + ); + } + catch (Exception e) { + try { + curator.delete().guaranteed().forPath(path); + } + catch (Exception e1) { + log.error(e1, "Failed to delete zNode[%s], but ignoring exception.", path); + } + + log.makeAlert(e, "Segment load/unload: uncaught exception.") + .addData("node", path) + .addData("nodeProperties", request) + .emit(); + } + + break; + case CHILD_REMOVED: + log.info("zNode[%s] was removed", event.getData().getPath()); + break; + default: + log.info("Ignoring event[%s]", event); + } + } + } + ); + loadQueueCache.start(); + } + catch (Exception e) { + Throwables.propagateIfPossible(e, IOException.class); + throw Throwables.propagate(e); + } + + started = true; + } + } + + @LifecycleStop + public void stop() + { + log.info("Stopping ZkCoordinator for [%s]", me); + synchronized (lock) { + if (!started) { + return; + } + + try { + loadQueueCache.close(); + serverAnnouncer.unannounce(); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + finally { + loadQueueCache = null; + started = false; + } + } + } + + public boolean isStarted() + { + return started; + } + + public void loadLocalCache() + { + final long start = System.currentTimeMillis(); + File baseDir = config.getInfoDir(); + if (!baseDir.exists() && !config.getInfoDir().mkdirs()) { + return; + } + + List cachedSegments = Lists.newArrayList(); + File[] segmentsToLoad = baseDir.listFiles(); + int ignored = 0; + for (int i = 0; i < segmentsToLoad.length; i++) { + File file = segmentsToLoad[i]; + log.info("Loading segment cache file [%d/%d][%s].", i, segmentsToLoad.length, file); + try { + final DataSegment segment = jsonMapper.readValue(file, DataSegment.class); + + if (!segment.getIdentifier().equals(file.getName())) { + log.warn("Ignoring cache file[%s] for segment[%s].", file.getPath(), segment.getIdentifier()); + ignored++; + } else if (segmentManager.isSegmentCached(segment)) { + cachedSegments.add(segment); + } else { + log.warn("Unable to find cache file for %s. Deleting lookup entry", segment.getIdentifier()); + + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); + if (!segmentInfoCacheFile.delete()) { + log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); + } + } + } + catch (Exception e) { + log.makeAlert(e, "Failed to load segment from segmentInfo file") + .addData("file", file) + .emit(); + } + } + + if (ignored > 0) { + log.makeAlert("Ignored misnamed segment cache files on startup.") + .addData("numIgnored", ignored) + .emit(); + } + + addSegments( + cachedSegments, + new DataSegmentChangeCallback() + { + @Override + public void execute() + { + log.info("Cache load took %,d ms", System.currentTimeMillis() - start); + } + } + ); + } + + public DataSegmentChangeHandler getDataSegmentChangeHandler() + { + return ZkCoordinator.this; + } + + /** + * Load a single segment. If the segment is loaded successfully, this function simply returns. Otherwise it will + * throw a SegmentLoadingException + * + * @throws SegmentLoadingException + */ + private void loadSegment(DataSegment segment, DataSegmentChangeCallback callback) throws SegmentLoadingException + { + final boolean loaded; + try { + loaded = segmentManager.loadSegment(segment); + } + catch (Exception e) { + removeSegment(segment, callback); + throw new SegmentLoadingException(e, "Exception loading segment[%s]", segment.getIdentifier()); + } + + if (loaded) { + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); + if (!segmentInfoCacheFile.exists()) { + try { + jsonMapper.writeValue(segmentInfoCacheFile, segment); + } + catch (IOException e) { + removeSegment(segment, callback); + throw new SegmentLoadingException( + e, "Failed to write to disk segment info cache file[%s]", segmentInfoCacheFile + ); + } + } + } + } + + @Override + public void addSegment(DataSegment segment, DataSegmentChangeCallback callback) + { + try { + log.info("Loading segment %s", segment.getIdentifier()); + /* + The lock below is used to prevent a race condition when the scheduled runnable in removeSegment() starts, + and if (segmentsToDelete.remove(segment)) returns true, in which case historical will start deleting segment + files. At that point, it's possible that right after the "if" check, addSegment() is called and actually loads + the segment, which makes dropping segment and downloading segment happen at the same time. + */ + if (segmentsToDelete.contains(segment)) { + /* + Both contains(segment) and remove(segment) can be moved inside the synchronized block. However, in that case, + each time when addSegment() is called, it has to wait for the lock in order to make progress, which will make + things slow. Given that in most cases segmentsToDelete.contains(segment) returns false, it will save a lot of + cost of acquiring lock by doing the "contains" check outside the synchronized block. + */ + synchronized (lock) { + segmentsToDelete.remove(segment); + } + } + loadSegment(segment, callback); + try { + announcer.announceSegment(segment); + } + catch (IOException e) { + throw new SegmentLoadingException(e, "Failed to announce segment[%s]", segment.getIdentifier()); + } + } + catch (SegmentLoadingException e) { + log.makeAlert(e, "Failed to load segment for dataSource") + .addData("segment", segment) + .emit(); + } + finally { + callback.execute(); + } + } + + private void addSegments(Collection segments, final DataSegmentChangeCallback callback) + { + ExecutorService loadingExecutor = null; + try (final BackgroundSegmentAnnouncer backgroundSegmentAnnouncer = + new BackgroundSegmentAnnouncer(announcer, exec, config.getAnnounceIntervalMillis())) { + + backgroundSegmentAnnouncer.startAnnouncing(); + + loadingExecutor = Execs.multiThreaded(config.getNumBootstrapThreads(), "ZkCoordinator-loading-%s"); + + final int numSegments = segments.size(); + final CountDownLatch latch = new CountDownLatch(numSegments); + final AtomicInteger counter = new AtomicInteger(0); + final CopyOnWriteArrayList failedSegments = new CopyOnWriteArrayList<>(); + for (final DataSegment segment : segments) { + loadingExecutor.submit( + new Runnable() + { + @Override + public void run() + { + try { + log.info( + "Loading segment[%d/%d][%s]", + counter.getAndIncrement(), + numSegments, + segment.getIdentifier() + ); + loadSegment(segment, callback); + try { + backgroundSegmentAnnouncer.announceSegment(segment); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SegmentLoadingException(e, "Loading Interrupted"); + } + } + catch (SegmentLoadingException e) { + log.error(e, "[%s] failed to load", segment.getIdentifier()); + failedSegments.add(segment); + } + finally { + latch.countDown(); + } + } + } + ); + } + + try { + latch.await(); + + if (failedSegments.size() > 0) { + log.makeAlert("%,d errors seen while loading segments", failedSegments.size()) + .addData("failedSegments", failedSegments); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.makeAlert(e, "LoadingInterrupted"); + } + + backgroundSegmentAnnouncer.finishAnnouncing(); + } + catch (SegmentLoadingException e) { + log.makeAlert(e, "Failed to load segments -- likely problem with announcing.") + .addData("numSegments", segments.size()) + .emit(); + } + finally { + callback.execute(); + if (loadingExecutor != null) { + loadingExecutor.shutdownNow(); + } + } + } + + + @Override + public void removeSegment(final DataSegment segment, final DataSegmentChangeCallback callback) + { + try { + announcer.unannounceSegment(segment); + segmentsToDelete.add(segment); + + log.info("Completely removing [%s] in [%,d] millis", segment.getIdentifier(), config.getDropSegmentDelayMillis()); + exec.schedule( + new Runnable() + { + @Override + public void run() + { + try { + synchronized (lock) { + if (segmentsToDelete.remove(segment)) { + segmentManager.dropSegment(segment); + + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); + if (!segmentInfoCacheFile.delete()) { + log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); + } + } + } + } + catch (Exception e) { + log.makeAlert(e, "Failed to remove segment! Possible resource leak!") + .addData("segment", segment) + .emit(); + } + } + }, + config.getDropSegmentDelayMillis(), + TimeUnit.MILLISECONDS + ); + } + catch (Exception e) { + log.makeAlert(e, "Failed to remove segment") + .addData("segment", segment) + .emit(); + } + finally { + callback.execute(); + } + } + + public Collection getPendingDeleteSnapshot() + { + return ImmutableList.copyOf(segmentsToDelete); + } + + private static class BackgroundSegmentAnnouncer implements AutoCloseable + { + private static final EmittingLogger log = new EmittingLogger(BackgroundSegmentAnnouncer.class); + + private final int intervalMillis; + private final DataSegmentAnnouncer announcer; + private final ScheduledExecutorService exec; + private final LinkedBlockingQueue queue; + private final SettableFuture doneAnnouncing; + + private final Object lock = new Object(); + + private volatile boolean finished = false; + private volatile ScheduledFuture startedAnnouncing = null; + private volatile ScheduledFuture nextAnnoucement = null; + + public BackgroundSegmentAnnouncer( + DataSegmentAnnouncer announcer, + ScheduledExecutorService exec, + int intervalMillis + ) + { + this.announcer = announcer; + this.exec = exec; + this.intervalMillis = intervalMillis; + this.queue = Queues.newLinkedBlockingQueue(); + this.doneAnnouncing = SettableFuture.create(); + } + + public void announceSegment(final DataSegment segment) throws InterruptedException + { + if (finished) { + throw new ISE("Announce segment called after finishAnnouncing"); + } + queue.put(segment); + } + + public void startAnnouncing() + { + if (intervalMillis <= 0) { + return; + } + + log.info("Starting background segment announcing task"); + + // schedule background announcing task + nextAnnoucement = startedAnnouncing = exec.schedule( + new Runnable() + { + @Override + public void run() + { + synchronized (lock) { + try { + if (!(finished && queue.isEmpty())) { + final List segments = Lists.newLinkedList(); + queue.drainTo(segments); + try { + announcer.announceSegments(segments); + nextAnnoucement = exec.schedule(this, intervalMillis, TimeUnit.MILLISECONDS); + } + catch (IOException e) { + doneAnnouncing.setException( + new SegmentLoadingException(e, "Failed to announce segments[%s]", segments) + ); + } + } else { + doneAnnouncing.set(true); + } + } + catch (Exception e) { + doneAnnouncing.setException(e); + } + } + } + }, + intervalMillis, + TimeUnit.MILLISECONDS + ); + } + + public void finishAnnouncing() throws SegmentLoadingException + { + synchronized (lock) { + finished = true; + // announce any remaining segments + try { + final List segments = Lists.newLinkedList(); + queue.drainTo(segments); + announcer.announceSegments(segments); + } + catch (Exception e) { + throw new SegmentLoadingException(e, "Failed to announce segments[%s]", queue); + } + + // get any exception that may have been thrown in background announcing + try { + // check in case intervalMillis is <= 0 + if (startedAnnouncing != null) { + startedAnnouncing.cancel(false); + } + // - if the task is waiting on the lock, then the queue will be empty by the time it runs + // - if the task just released it, then the lock ensures any exception is set in doneAnnouncing + if (doneAnnouncing.isDone()) { + doneAnnouncing.get(); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new SegmentLoadingException(e, "Loading Interrupted"); + } + catch (ExecutionException e) { + throw new SegmentLoadingException(e.getCause(), "Background Announcing Task Failed"); + } + } + log.info("Completed background segment announcing"); + } + + @Override + public void close() + { + // stop background scheduling + synchronized (lock) { + finished = true; + if (nextAnnoucement != null) { + nextAnnoucement.cancel(false); + } + } + } + } +} From eb37b91a2d59ff01c3108881f538c6140485dffe Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Mon, 11 Sep 2017 10:34:33 -0500 Subject: [PATCH 6/9] remove non-zk part from ZkCoordinator --- .../server/coordination/ZkCoordinator.java | 447 +----------------- 1 file changed, 6 insertions(+), 441 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java index 4ae8887694eb..de07f5f19637 100644 --- a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java +++ b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java @@ -21,22 +21,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.collect.Queues; -import com.google.common.util.concurrent.SettableFuture; import com.google.inject.Inject; import com.metamx.emitter.EmittingLogger; import io.druid.concurrent.Execs; -import io.druid.java.util.common.ISE; -import io.druid.java.util.common.concurrent.ScheduledExecutorFactory; import io.druid.java.util.common.lifecycle.LifecycleStart; import io.druid.java.util.common.lifecycle.LifecycleStop; import io.druid.segment.loading.SegmentLoaderConfig; -import io.druid.segment.loading.SegmentLoadingException; -import io.druid.server.SegmentManager; import io.druid.server.initialization.ZkPathsConfig; -import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCache; @@ -44,68 +35,42 @@ import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.utils.ZKPaths; -import java.io.File; import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; /** */ -public class ZkCoordinator implements DataSegmentChangeHandler +public class ZkCoordinator { private static final EmittingLogger log = new EmittingLogger(ZkCoordinator.class); private final Object lock = new Object(); + private final SimpleDataSegmentChangeHandler dataSegmentChangeHandler; private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPaths; private final SegmentLoaderConfig config; private final DruidServerMetadata me; private final CuratorFramework curator; - private final DataSegmentAnnouncer announcer; - private final DataSegmentServerAnnouncer serverAnnouncer; - private final SegmentManager segmentManager; - private final ScheduledExecutorService exec; - private final ConcurrentSkipListSet segmentsToDelete; - private volatile PathChildrenCache loadQueueCache; private volatile boolean started = false; @Inject public ZkCoordinator( + SimpleDataSegmentChangeHandler dataSegmentChangeHandler, ObjectMapper jsonMapper, SegmentLoaderConfig config, ZkPathsConfig zkPaths, DruidServerMetadata me, - DataSegmentAnnouncer announcer, - DataSegmentServerAnnouncer serverAnnouncer, - CuratorFramework curator, - SegmentManager segmentManager, - ScheduledExecutorFactory factory + CuratorFramework curator ) { + this.dataSegmentChangeHandler = dataSegmentChangeHandler; this.jsonMapper = jsonMapper; this.zkPaths = zkPaths; this.config = config; this.me = me; this.curator = curator; - this.announcer = announcer; - this.serverAnnouncer = serverAnnouncer; - this.segmentManager = segmentManager; - - this.exec = factory.create(1, "ZkCoordinator-Exec--%d"); - this.segmentsToDelete = new ConcurrentSkipListSet<>(); } @LifecycleStart @@ -135,9 +100,6 @@ public void start() throws IOException curator.newNamespaceAwareEnsurePath(servedSegmentsLocation).ensure(curator.getZookeeperClient()); curator.newNamespaceAwareEnsurePath(liveSegmentsLocation).ensure(curator.getZookeeperClient()); - loadLocalCache(); - serverAnnouncer.announce(); - loadQueueCache.getListenable().addListener( new PathChildrenCacheListener() { @@ -156,7 +118,7 @@ public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) th try { request.go( - getDataSegmentChangeHandler(), + dataSegmentChangeHandler, new DataSegmentChangeCallback() { boolean hasRun = false; @@ -231,7 +193,6 @@ public void stop() try { loadQueueCache.close(); - serverAnnouncer.unannounce(); } catch (Exception e) { throw Throwables.propagate(e); @@ -247,400 +208,4 @@ public boolean isStarted() { return started; } - - public void loadLocalCache() - { - final long start = System.currentTimeMillis(); - File baseDir = config.getInfoDir(); - if (!baseDir.exists() && !config.getInfoDir().mkdirs()) { - return; - } - - List cachedSegments = Lists.newArrayList(); - File[] segmentsToLoad = baseDir.listFiles(); - int ignored = 0; - for (int i = 0; i < segmentsToLoad.length; i++) { - File file = segmentsToLoad[i]; - log.info("Loading segment cache file [%d/%d][%s].", i, segmentsToLoad.length, file); - try { - final DataSegment segment = jsonMapper.readValue(file, DataSegment.class); - - if (!segment.getIdentifier().equals(file.getName())) { - log.warn("Ignoring cache file[%s] for segment[%s].", file.getPath(), segment.getIdentifier()); - ignored++; - } else if (segmentManager.isSegmentCached(segment)) { - cachedSegments.add(segment); - } else { - log.warn("Unable to find cache file for %s. Deleting lookup entry", segment.getIdentifier()); - - File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); - if (!segmentInfoCacheFile.delete()) { - log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); - } - } - } - catch (Exception e) { - log.makeAlert(e, "Failed to load segment from segmentInfo file") - .addData("file", file) - .emit(); - } - } - - if (ignored > 0) { - log.makeAlert("Ignored misnamed segment cache files on startup.") - .addData("numIgnored", ignored) - .emit(); - } - - addSegments( - cachedSegments, - new DataSegmentChangeCallback() - { - @Override - public void execute() - { - log.info("Cache load took %,d ms", System.currentTimeMillis() - start); - } - } - ); - } - - public DataSegmentChangeHandler getDataSegmentChangeHandler() - { - return ZkCoordinator.this; - } - - /** - * Load a single segment. If the segment is loaded successfully, this function simply returns. Otherwise it will - * throw a SegmentLoadingException - * - * @throws SegmentLoadingException - */ - private void loadSegment(DataSegment segment, DataSegmentChangeCallback callback) throws SegmentLoadingException - { - final boolean loaded; - try { - loaded = segmentManager.loadSegment(segment); - } - catch (Exception e) { - removeSegment(segment, callback); - throw new SegmentLoadingException(e, "Exception loading segment[%s]", segment.getIdentifier()); - } - - if (loaded) { - File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); - if (!segmentInfoCacheFile.exists()) { - try { - jsonMapper.writeValue(segmentInfoCacheFile, segment); - } - catch (IOException e) { - removeSegment(segment, callback); - throw new SegmentLoadingException( - e, "Failed to write to disk segment info cache file[%s]", segmentInfoCacheFile - ); - } - } - } - } - - @Override - public void addSegment(DataSegment segment, DataSegmentChangeCallback callback) - { - try { - log.info("Loading segment %s", segment.getIdentifier()); - /* - The lock below is used to prevent a race condition when the scheduled runnable in removeSegment() starts, - and if (segmentsToDelete.remove(segment)) returns true, in which case historical will start deleting segment - files. At that point, it's possible that right after the "if" check, addSegment() is called and actually loads - the segment, which makes dropping segment and downloading segment happen at the same time. - */ - if (segmentsToDelete.contains(segment)) { - /* - Both contains(segment) and remove(segment) can be moved inside the synchronized block. However, in that case, - each time when addSegment() is called, it has to wait for the lock in order to make progress, which will make - things slow. Given that in most cases segmentsToDelete.contains(segment) returns false, it will save a lot of - cost of acquiring lock by doing the "contains" check outside the synchronized block. - */ - synchronized (lock) { - segmentsToDelete.remove(segment); - } - } - loadSegment(segment, callback); - try { - announcer.announceSegment(segment); - } - catch (IOException e) { - throw new SegmentLoadingException(e, "Failed to announce segment[%s]", segment.getIdentifier()); - } - } - catch (SegmentLoadingException e) { - log.makeAlert(e, "Failed to load segment for dataSource") - .addData("segment", segment) - .emit(); - } - finally { - callback.execute(); - } - } - - private void addSegments(Collection segments, final DataSegmentChangeCallback callback) - { - ExecutorService loadingExecutor = null; - try (final BackgroundSegmentAnnouncer backgroundSegmentAnnouncer = - new BackgroundSegmentAnnouncer(announcer, exec, config.getAnnounceIntervalMillis())) { - - backgroundSegmentAnnouncer.startAnnouncing(); - - loadingExecutor = Execs.multiThreaded(config.getNumBootstrapThreads(), "ZkCoordinator-loading-%s"); - - final int numSegments = segments.size(); - final CountDownLatch latch = new CountDownLatch(numSegments); - final AtomicInteger counter = new AtomicInteger(0); - final CopyOnWriteArrayList failedSegments = new CopyOnWriteArrayList<>(); - for (final DataSegment segment : segments) { - loadingExecutor.submit( - new Runnable() - { - @Override - public void run() - { - try { - log.info( - "Loading segment[%d/%d][%s]", - counter.getAndIncrement(), - numSegments, - segment.getIdentifier() - ); - loadSegment(segment, callback); - try { - backgroundSegmentAnnouncer.announceSegment(segment); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SegmentLoadingException(e, "Loading Interrupted"); - } - } - catch (SegmentLoadingException e) { - log.error(e, "[%s] failed to load", segment.getIdentifier()); - failedSegments.add(segment); - } - finally { - latch.countDown(); - } - } - } - ); - } - - try { - latch.await(); - - if (failedSegments.size() > 0) { - log.makeAlert("%,d errors seen while loading segments", failedSegments.size()) - .addData("failedSegments", failedSegments); - } - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.makeAlert(e, "LoadingInterrupted"); - } - - backgroundSegmentAnnouncer.finishAnnouncing(); - } - catch (SegmentLoadingException e) { - log.makeAlert(e, "Failed to load segments -- likely problem with announcing.") - .addData("numSegments", segments.size()) - .emit(); - } - finally { - callback.execute(); - if (loadingExecutor != null) { - loadingExecutor.shutdownNow(); - } - } - } - - - @Override - public void removeSegment(final DataSegment segment, final DataSegmentChangeCallback callback) - { - try { - announcer.unannounceSegment(segment); - segmentsToDelete.add(segment); - - log.info("Completely removing [%s] in [%,d] millis", segment.getIdentifier(), config.getDropSegmentDelayMillis()); - exec.schedule( - new Runnable() - { - @Override - public void run() - { - try { - synchronized (lock) { - if (segmentsToDelete.remove(segment)) { - segmentManager.dropSegment(segment); - - File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); - if (!segmentInfoCacheFile.delete()) { - log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); - } - } - } - } - catch (Exception e) { - log.makeAlert(e, "Failed to remove segment! Possible resource leak!") - .addData("segment", segment) - .emit(); - } - } - }, - config.getDropSegmentDelayMillis(), - TimeUnit.MILLISECONDS - ); - } - catch (Exception e) { - log.makeAlert(e, "Failed to remove segment") - .addData("segment", segment) - .emit(); - } - finally { - callback.execute(); - } - } - - public Collection getPendingDeleteSnapshot() - { - return ImmutableList.copyOf(segmentsToDelete); - } - - private static class BackgroundSegmentAnnouncer implements AutoCloseable - { - private static final EmittingLogger log = new EmittingLogger(BackgroundSegmentAnnouncer.class); - - private final int intervalMillis; - private final DataSegmentAnnouncer announcer; - private final ScheduledExecutorService exec; - private final LinkedBlockingQueue queue; - private final SettableFuture doneAnnouncing; - - private final Object lock = new Object(); - - private volatile boolean finished = false; - private volatile ScheduledFuture startedAnnouncing = null; - private volatile ScheduledFuture nextAnnoucement = null; - - public BackgroundSegmentAnnouncer( - DataSegmentAnnouncer announcer, - ScheduledExecutorService exec, - int intervalMillis - ) - { - this.announcer = announcer; - this.exec = exec; - this.intervalMillis = intervalMillis; - this.queue = Queues.newLinkedBlockingQueue(); - this.doneAnnouncing = SettableFuture.create(); - } - - public void announceSegment(final DataSegment segment) throws InterruptedException - { - if (finished) { - throw new ISE("Announce segment called after finishAnnouncing"); - } - queue.put(segment); - } - - public void startAnnouncing() - { - if (intervalMillis <= 0) { - return; - } - - log.info("Starting background segment announcing task"); - - // schedule background announcing task - nextAnnoucement = startedAnnouncing = exec.schedule( - new Runnable() - { - @Override - public void run() - { - synchronized (lock) { - try { - if (!(finished && queue.isEmpty())) { - final List segments = Lists.newLinkedList(); - queue.drainTo(segments); - try { - announcer.announceSegments(segments); - nextAnnoucement = exec.schedule(this, intervalMillis, TimeUnit.MILLISECONDS); - } - catch (IOException e) { - doneAnnouncing.setException( - new SegmentLoadingException(e, "Failed to announce segments[%s]", segments) - ); - } - } else { - doneAnnouncing.set(true); - } - } - catch (Exception e) { - doneAnnouncing.setException(e); - } - } - } - }, - intervalMillis, - TimeUnit.MILLISECONDS - ); - } - - public void finishAnnouncing() throws SegmentLoadingException - { - synchronized (lock) { - finished = true; - // announce any remaining segments - try { - final List segments = Lists.newLinkedList(); - queue.drainTo(segments); - announcer.announceSegments(segments); - } - catch (Exception e) { - throw new SegmentLoadingException(e, "Failed to announce segments[%s]", queue); - } - - // get any exception that may have been thrown in background announcing - try { - // check in case intervalMillis is <= 0 - if (startedAnnouncing != null) { - startedAnnouncing.cancel(false); - } - // - if the task is waiting on the lock, then the queue will be empty by the time it runs - // - if the task just released it, then the lock ensures any exception is set in doneAnnouncing - if (doneAnnouncing.isDone()) { - doneAnnouncing.get(); - } - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new SegmentLoadingException(e, "Loading Interrupted"); - } - catch (ExecutionException e) { - throw new SegmentLoadingException(e.getCause(), "Background Announcing Task Failed"); - } - } - log.info("Completed background segment announcing"); - } - - @Override - public void close() - { - // stop background scheduling - synchronized (lock) { - finished = true; - if (nextAnnoucement != null) { - nextAnnoucement.cancel(false); - } - } - } - } } From 8a7f0a4b95499eba640755b3e3ab1a730c925b11 Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Wed, 27 Sep 2017 13:14:10 -0500 Subject: [PATCH 7/9] remove zk part from SegmentLoadDropHandler --- .../coordination/SegmentLoadDropHandler.java | 135 ++---------------- 1 file changed, 15 insertions(+), 120 deletions(-) diff --git a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java index ed787f692add..7e52beaea287 100644 --- a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java @@ -34,14 +34,7 @@ import io.druid.segment.loading.SegmentLoaderConfig; import io.druid.segment.loading.SegmentLoadingException; import io.druid.server.SegmentManager; -import io.druid.server.initialization.ZkPathsConfig; import io.druid.timeline.DataSegment; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; -import org.apache.curator.utils.ZKPaths; import java.io.File; import java.io.IOException; @@ -60,50 +53,44 @@ /** */ -public class ZkCoordinator implements DataSegmentChangeHandler +public class SimpleDataSegmentChangeHandler implements DataSegmentChangeHandler { - private static final EmittingLogger log = new EmittingLogger(ZkCoordinator.class); + private static final EmittingLogger log = new EmittingLogger(SimpleDataSegmentChangeHandler.class); private final Object lock = new Object(); private final ObjectMapper jsonMapper; - private final ZkPathsConfig zkPaths; private final SegmentLoaderConfig config; - private final DruidServerMetadata me; - private final CuratorFramework curator; + private final DruidServerMetadata me; //unused ? private final DataSegmentAnnouncer announcer; private final DataSegmentServerAnnouncer serverAnnouncer; private final SegmentManager segmentManager; private final ScheduledExecutorService exec; private final ConcurrentSkipListSet segmentsToDelete; - - private volatile PathChildrenCache loadQueueCache; private volatile boolean started = false; + //TODO: this also needs to have concept of asynchrounous add/remove segments that would be used in the SegmentListerResource. + @Inject - public ZkCoordinator( + public SimpleDataSegmentChangeHandler( ObjectMapper jsonMapper, SegmentLoaderConfig config, - ZkPathsConfig zkPaths, DruidServerMetadata me, DataSegmentAnnouncer announcer, DataSegmentServerAnnouncer serverAnnouncer, - CuratorFramework curator, SegmentManager segmentManager, ScheduledExecutorFactory factory ) { this.jsonMapper = jsonMapper; - this.zkPaths = zkPaths; this.config = config; this.me = me; - this.curator = curator; this.announcer = announcer; this.serverAnnouncer = serverAnnouncer; this.segmentManager = segmentManager; - this.exec = factory.create(1, "ZkCoordinator-Exec--%d"); + this.exec = factory.create(1, "SimpleDataSegmentChangeHandler-Exec--%d"); this.segmentsToDelete = new ConcurrentSkipListSet<>(); } @@ -115,106 +102,21 @@ public void start() throws IOException return; } - log.info("Starting zkCoordinator for server[%s]", me.getName()); - - final String loadQueueLocation = ZKPaths.makePath(zkPaths.getLoadQueuePath(), me.getName()); - final String servedSegmentsLocation = ZKPaths.makePath(zkPaths.getServedSegmentsPath(), me.getName()); - final String liveSegmentsLocation = ZKPaths.makePath(zkPaths.getLiveSegmentsPath(), me.getName()); - - loadQueueCache = new PathChildrenCache( - curator, - loadQueueLocation, - true, - true, - Execs.multiThreaded(config.getNumLoadingThreads(), "ZkCoordinator-%s") - ); - try { - curator.newNamespaceAwareEnsurePath(loadQueueLocation).ensure(curator.getZookeeperClient()); - curator.newNamespaceAwareEnsurePath(servedSegmentsLocation).ensure(curator.getZookeeperClient()); - curator.newNamespaceAwareEnsurePath(liveSegmentsLocation).ensure(curator.getZookeeperClient()); - loadLocalCache(); - serverAnnouncer.announce(); - loadQueueCache.getListenable().addListener( - new PathChildrenCacheListener() - { - @Override - public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception - { - final ChildData child = event.getData(); - switch (event.getType()) { - case CHILD_ADDED: - final String path = child.getPath(); - final DataSegmentChangeRequest request = jsonMapper.readValue( - child.getData(), DataSegmentChangeRequest.class - ); - - log.info("New request[%s] with zNode[%s].", request.asString(), path); - - try { - request.go( - getDataSegmentChangeHandler(), - new DataSegmentChangeCallback() - { - boolean hasRun = false; - - @Override - public void execute() - { - try { - if (!hasRun) { - curator.delete().guaranteed().forPath(path); - log.info("Completed request [%s]", request.asString()); - hasRun = true; - } - } - catch (Exception e) { - try { - curator.delete().guaranteed().forPath(path); - } - catch (Exception e1) { - log.error(e1, "Failed to delete zNode[%s], but ignoring exception.", path); - } - log.error(e, "Exception while removing zNode[%s]", path); - throw Throwables.propagate(e); - } - } - } - ); - } - catch (Exception e) { - try { - curator.delete().guaranteed().forPath(path); - } - catch (Exception e1) { - log.error(e1, "Failed to delete zNode[%s], but ignoring exception.", path); - } - - log.makeAlert(e, "Segment load/unload: uncaught exception.") - .addData("node", path) - .addData("nodeProperties", request) - .emit(); - } - - break; - case CHILD_REMOVED: - log.info("zNode[%s] was removed", event.getData().getPath()); - break; - default: - log.info("Ignoring event[%s]", event); - } - } - } - ); - loadQueueCache.start(); + //TODO: announce this historical here too, is this class used at realtime nodes too, then we have two locations + //where announcement happens? + //lifecycle driven, will all peons run this code? Maybe ZkCoordinator was not configured to run on realtime nodes + //but then how were they listening for segment load requests? + //however since announcement is lifecycle.last so DiscoveryDruidNode announcement doesn't have to be done here + //explicitly? + serverAnnouncer.announce(); } catch (Exception e) { Throwables.propagateIfPossible(e, IOException.class); throw Throwables.propagate(e); } - started = true; } } @@ -222,21 +124,19 @@ public void execute() @LifecycleStop public void stop() { - log.info("Stopping ZkCoordinator for [%s]", me); synchronized (lock) { if (!started) { return; } try { - loadQueueCache.close(); serverAnnouncer.unannounce(); + //TODO: should we unannounce the server here? } catch (Exception e) { throw Throwables.propagate(e); } finally { - loadQueueCache = null; started = false; } } @@ -304,11 +204,6 @@ public void execute() ); } - public DataSegmentChangeHandler getDataSegmentChangeHandler() - { - return ZkCoordinator.this; - } - /** * Load a single segment. If the segment is loaded successfully, this function simply returns. Otherwise it will * throw a SegmentLoadingException From 11d21edca5e1ef577402ebfe2172882bd16bf2ad Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Mon, 11 Sep 2017 10:20:42 -0500 Subject: [PATCH 8/9] additional changes for segment load/drop management with http --- .../java/io/druid/java/util/common/Pair.java | 9 +- .../io/druid/client/ImmutableDruidServer.java | 10 + .../segment/loading/SegmentLoaderConfig.java | 8 + .../DataSegmentChangeCallback.java | 2 + .../SegmentChangeRequestDrop.java | 21 + .../SegmentChangeRequestLoad.java | 21 + .../coordination/SegmentLoadDropHandler.java | 371 ++++++++++-- .../server/coordination/ZkCoordinator.java | 12 +- .../server/coordinator/DruidCoordinator.java | 5 +- .../coordinator/DruidCoordinatorConfig.java | 24 + .../server/coordinator/HttpLoadQueuePeon.java | 477 ++++++++++++++-- .../server/coordinator/LoadQueuePeon.java | 3 + .../coordinator/LoadQueueTaskMaster.java | 31 +- .../server/http/SegmentListerResource.java | 126 +++- .../metrics/HistoricalMetricsMonitor.java | 10 +- .../SegmentLoadDropHandlerTest.java | 483 ++++++++++++++++ .../coordination/ZkCoordinatorTest.java | 537 +++--------------- .../coordinator/DruidCoordinatorTest.java | 2 +- .../coordinator/HttpLoadQueuePeonTest.java | 210 +++++++ .../server/coordinator/LoadQueuePeonTest.java | 4 +- .../coordinator/LoadQueuePeonTester.java | 2 +- .../metrics/HistoricalMetricsMonitorTest.java | 14 +- .../java/io/druid/cli/CliCoordinator.java | 9 +- .../src/main/java/io/druid/cli/CliPeon.java | 18 + 24 files changed, 1801 insertions(+), 608 deletions(-) create mode 100644 server/src/test/java/io/druid/server/coordination/SegmentLoadDropHandlerTest.java create mode 100644 server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java diff --git a/java-util/src/main/java/io/druid/java/util/common/Pair.java b/java-util/src/main/java/io/druid/java/util/common/Pair.java index 456b41970bd3..f821eb599342 100644 --- a/java-util/src/main/java/io/druid/java/util/common/Pair.java +++ b/java-util/src/main/java/io/druid/java/util/common/Pair.java @@ -19,6 +19,7 @@ package io.druid.java.util.common; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import java.util.Comparator; @@ -33,10 +34,16 @@ public static Pair of(T1 lhs, T2 rhs) return new Pair<>(lhs, rhs); } + @JsonProperty public final T1 lhs; + + @JsonProperty public final T2 rhs; - public Pair(T1 lhs, T2 rhs) + public Pair( + @JsonProperty("lhs") T1 lhs, + @JsonProperty("rhs") T2 rhs + ) { this.lhs = lhs; this.rhs = rhs; diff --git a/server/src/main/java/io/druid/client/ImmutableDruidServer.java b/server/src/main/java/io/druid/client/ImmutableDruidServer.java index 1851befe277e..af36ec9563cd 100644 --- a/server/src/main/java/io/druid/client/ImmutableDruidServer.java +++ b/server/src/main/java/io/druid/client/ImmutableDruidServer.java @@ -20,6 +20,7 @@ package io.druid.client; import com.google.common.collect.ImmutableMap; +import com.metamx.common.StringUtils; import io.druid.server.coordination.DruidServerMetadata; import io.druid.server.coordination.ServerType; import io.druid.timeline.DataSegment; @@ -108,6 +109,15 @@ public Map getSegments() return segments; } + public String getURL() + { + if (metadata.getHostAndTlsPort() != null) { + return StringUtils.safeFormat("https://%s", metadata.getHostAndTlsPort()); + } else { + return StringUtils.safeFormat("http://%s", metadata.getHostAndPort()); + } + } + @Override public String toString() { diff --git a/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java b/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java index f67cd649bfa0..87131bf103dd 100644 --- a/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java +++ b/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java @@ -53,6 +53,9 @@ public class SegmentLoaderConfig @JsonProperty private File infoDir = null; + @JsonProperty + private int statusQueueMaxSize = 100; + public List getLocations() { return locations; @@ -96,6 +99,11 @@ public File getInfoDir() return infoDir; } + public int getStatusQueueMaxSize() + { + return statusQueueMaxSize; + } + public SegmentLoaderConfig withLocations(List locations) { SegmentLoaderConfig retVal = new SegmentLoaderConfig(); diff --git a/server/src/main/java/io/druid/server/coordination/DataSegmentChangeCallback.java b/server/src/main/java/io/druid/server/coordination/DataSegmentChangeCallback.java index a21369976b5f..82dabc097ab1 100644 --- a/server/src/main/java/io/druid/server/coordination/DataSegmentChangeCallback.java +++ b/server/src/main/java/io/druid/server/coordination/DataSegmentChangeCallback.java @@ -24,4 +24,6 @@ public interface DataSegmentChangeCallback { public void execute(); + + DataSegmentChangeCallback NOOP = () -> {}; } diff --git a/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestDrop.java b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestDrop.java index 5cb7c18d3a67..86cc31e401b0 100644 --- a/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestDrop.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestDrop.java @@ -25,6 +25,8 @@ import io.druid.java.util.common.StringUtils; import io.druid.timeline.DataSegment; +import java.util.Objects; + /** */ public class SegmentChangeRequestDrop implements DataSegmentChangeRequest @@ -58,6 +60,25 @@ public String asString() return StringUtils.format("DROP: %s", segment.getIdentifier()); } + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SegmentChangeRequestDrop that = (SegmentChangeRequestDrop) o; + return Objects.equals(segment, that.segment); + } + + @Override + public int hashCode() + { + return Objects.hash(segment); + } + @Override public String toString() { diff --git a/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestLoad.java b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestLoad.java index 9ad6f667f6a8..24a73f63a94a 100644 --- a/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestLoad.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestLoad.java @@ -25,6 +25,8 @@ import io.druid.java.util.common.StringUtils; import io.druid.timeline.DataSegment; +import java.util.Objects; + /** */ public class SegmentChangeRequestLoad implements DataSegmentChangeRequest @@ -58,6 +60,25 @@ public String asString() return StringUtils.format("LOAD: %s", segment.getIdentifier()); } + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + SegmentChangeRequestLoad that = (SegmentChangeRequestLoad) o; + return Objects.equals(segment, that.segment); + } + + @Override + public int hashCode() + { + return Objects.hash(segment); + } + @Override public String toString() { diff --git a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java index 7e52beaea287..8aada51f65d6 100644 --- a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java @@ -18,17 +18,27 @@ */ package io.druid.server.coordination; + +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.base.Throwables; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Queues; +import com.google.common.util.concurrent.AbstractFuture; +import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.inject.Inject; import com.metamx.emitter.EmittingLogger; import io.druid.concurrent.Execs; +import io.druid.guice.ManageLifecycle; +import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; -import io.druid.java.util.common.concurrent.ScheduledExecutorFactory; +import io.druid.java.util.common.Pair; import io.druid.java.util.common.lifecycle.LifecycleStart; import io.druid.java.util.common.lifecycle.LifecycleStop; import io.druid.segment.loading.SegmentLoaderConfig; @@ -38,30 +48,36 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; /** */ -public class SimpleDataSegmentChangeHandler implements DataSegmentChangeHandler +@ManageLifecycle +public class SegmentLoadDropHandler { - private static final EmittingLogger log = new EmittingLogger(SimpleDataSegmentChangeHandler.class); + private static final EmittingLogger log = new EmittingLogger(SegmentLoadDropHandler.class); private final Object lock = new Object(); private final ObjectMapper jsonMapper; private final SegmentLoaderConfig config; - private final DruidServerMetadata me; //unused ? private final DataSegmentAnnouncer announcer; private final DataSegmentServerAnnouncer serverAnnouncer; private final SegmentManager segmentManager; @@ -70,28 +86,53 @@ public class SimpleDataSegmentChangeHandler implements DataSegmentChangeHandler private volatile boolean started = false; - //TODO: this also needs to have concept of asynchrounous add/remove segments that would be used in the SegmentListerResource. + // Keep history of load/drop request status in a LRU cache to maintain idempotency if same request shows up + // again and to return status of a completed request. Maximum size of this cache must be significantly greater + // than number of pending load/drop requests. so that history is not lost too quickly. + private final Cache> requestStatuses; + private final Object requestStatusesLock = new Object(); + + // This is the list of unresolved futures returned to callers of processBatch(List) + // Threads loading/dropping segments resolve these futures as and when some segment load/drop finishes. + private final LinkedHashSet waitingFutures = new LinkedHashSet<>(); @Inject - public SimpleDataSegmentChangeHandler( + public SegmentLoadDropHandler( + ObjectMapper jsonMapper, + SegmentLoaderConfig config, + DataSegmentAnnouncer announcer, + DataSegmentServerAnnouncer serverAnnouncer, + SegmentManager segmentManager + ) + { + this(jsonMapper, config, announcer, serverAnnouncer, segmentManager, + Executors.newScheduledThreadPool( + config.getNumLoadingThreads(), + Execs.makeThreadFactory("SimpleDataSegmentChangeHandler-%s") + ) + ); + } + + @VisibleForTesting + SegmentLoadDropHandler( ObjectMapper jsonMapper, SegmentLoaderConfig config, - DruidServerMetadata me, DataSegmentAnnouncer announcer, DataSegmentServerAnnouncer serverAnnouncer, SegmentManager segmentManager, - ScheduledExecutorFactory factory + ScheduledExecutorService exec ) { this.jsonMapper = jsonMapper; this.config = config; - this.me = me; this.announcer = announcer; this.serverAnnouncer = serverAnnouncer; this.segmentManager = segmentManager; - this.exec = factory.create(1, "SimpleDataSegmentChangeHandler-Exec--%d"); + this.exec = exec; this.segmentsToDelete = new ConcurrentSkipListSet<>(); + + requestStatuses = CacheBuilder.newBuilder().maximumSize(config.getStatusQueueMaxSize()).initialCapacity(8).build(); } @LifecycleStart @@ -102,15 +143,9 @@ public void start() throws IOException return; } + log.info("Starting..."); try { loadLocalCache(); - - //TODO: announce this historical here too, is this class used at realtime nodes too, then we have two locations - //where announcement happens? - //lifecycle driven, will all peons run this code? Maybe ZkCoordinator was not configured to run on realtime nodes - //but then how were they listening for segment load requests? - //however since announcement is lifecycle.last so DiscoveryDruidNode announcement doesn't have to be done here - //explicitly? serverAnnouncer.announce(); } catch (Exception e) { @@ -118,6 +153,7 @@ public void start() throws IOException throw Throwables.propagate(e); } started = true; + log.info("Started."); } } @@ -129,9 +165,9 @@ public void stop() return; } + log.info("Stopping..."); try { serverAnnouncer.unannounce(); - //TODO: should we unannounce the server here? } catch (Exception e) { throw Throwables.propagate(e); @@ -139,6 +175,7 @@ public void stop() finally { started = false; } + log.info("Stopped."); } } @@ -147,12 +184,16 @@ public boolean isStarted() return started; } - public void loadLocalCache() + private void loadLocalCache() { final long start = System.currentTimeMillis(); File baseDir = config.getInfoDir(); - if (!baseDir.exists() && !config.getInfoDir().mkdirs()) { - return; + if (!baseDir.isDirectory()) { + if (baseDir.exists()) { + throw new ISE("[%s] exists but not a directory.", baseDir); + } else if (!baseDir.mkdirs()) { + throw new ISE("Failed to create directory[%s].", baseDir); + } } List cachedSegments = Lists.newArrayList(); @@ -172,7 +213,7 @@ public void loadLocalCache() } else { log.warn("Unable to find cache file for %s. Deleting lookup entry", segment.getIdentifier()); - File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); + File segmentInfoCacheFile = new File(baseDir, segment.getIdentifier()); if (!segmentInfoCacheFile.delete()) { log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); } @@ -217,7 +258,7 @@ private void loadSegment(DataSegment segment, DataSegmentChangeCallback callback loaded = segmentManager.loadSegment(segment); } catch (Exception e) { - removeSegment(segment, callback); + removeSegment(segment, callback, false); throw new SegmentLoadingException(e, "Exception loading segment[%s]", segment.getIdentifier()); } @@ -228,7 +269,7 @@ private void loadSegment(DataSegment segment, DataSegmentChangeCallback callback jsonMapper.writeValue(segmentInfoCacheFile, segment); } catch (IOException e) { - removeSegment(segment, callback); + removeSegment(segment, callback, false); throw new SegmentLoadingException( e, "Failed to write to disk segment info cache file[%s]", segmentInfoCacheFile ); @@ -237,9 +278,9 @@ private void loadSegment(DataSegment segment, DataSegmentChangeCallback callback } } - @Override - public void addSegment(DataSegment segment, DataSegmentChangeCallback callback) + void addSegment(DataSegment segment, DataSegmentChangeCallback callback) { + Status result = null; try { log.info("Loading segment %s", segment.getIdentifier()); /* @@ -266,13 +307,17 @@ each time when addSegment() is called, it has to wait for the lock in order to m catch (IOException e) { throw new SegmentLoadingException(e, "Failed to announce segment[%s]", segment.getIdentifier()); } + + result = Status.SUCCESS; } - catch (SegmentLoadingException e) { + catch (Exception e) { log.makeAlert(e, "Failed to load segment for dataSource") .addData("segment", segment) .emit(); + result = Status.failed(e.getMessage()); } finally { + updateStatusAndResolveWaitingFutures(new SegmentChangeRequestLoad(segment), result); callback.execute(); } } @@ -285,7 +330,7 @@ private void addSegments(Collection segments, final DataSegmentChan backgroundSegmentAnnouncer.startAnnouncing(); - loadingExecutor = Execs.multiThreaded(config.getNumBootstrapThreads(), "ZkCoordinator-loading-%s"); + loadingExecutor = Execs.multiThreaded(config.getNumBootstrapThreads(), "Segment-Load-Startup-%s"); final int numSegments = segments.size(); final CountDownLatch latch = new CountDownLatch(numSegments); @@ -354,48 +399,67 @@ public void run() } } - - @Override - public void removeSegment(final DataSegment segment, final DataSegmentChangeCallback callback) + void removeSegment( + final DataSegment segment, + final DataSegmentChangeCallback callback, + final boolean scheduleDrop + ) { try { announcer.unannounceSegment(segment); segmentsToDelete.add(segment); - log.info("Completely removing [%s] in [%,d] millis", segment.getIdentifier(), config.getDropSegmentDelayMillis()); - exec.schedule( - new Runnable() - { - @Override - public void run() - { - try { - synchronized (lock) { - if (segmentsToDelete.remove(segment)) { - segmentManager.dropSegment(segment); - - File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); - if (!segmentInfoCacheFile.delete()) { - log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); - } - } + Runnable runnable = new Runnable() + { + @Override + public void run() + { + Status result = null; + try { + synchronized (lock) { + if (segmentsToDelete.remove(segment)) { + segmentManager.dropSegment(segment); + + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); + if (!segmentInfoCacheFile.delete()) { + log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); } } - catch (Exception e) { - log.makeAlert(e, "Failed to remove segment! Possible resource leak!") - .addData("segment", segment) - .emit(); - } } - }, - config.getDropSegmentDelayMillis(), - TimeUnit.MILLISECONDS - ); + + result = Status.SUCCESS; + } + catch (Exception e) { + log.makeAlert(e, "Failed to remove segment! Possible resource leak!") + .addData("segment", segment) + .emit(); + result = Status.failed(e.getMessage()); + } + finally { + updateStatusAndResolveWaitingFutures(new SegmentChangeRequestDrop(segment), result); + } + } + }; + + if (scheduleDrop) { + log.info( + "Completely removing [%s] in [%,d] millis", + segment.getIdentifier(), + config.getDropSegmentDelayMillis() + ); + exec.schedule( + runnable, + config.getDropSegmentDelayMillis(), + TimeUnit.MILLISECONDS + ); + } } catch (Exception e) { log.makeAlert(e, "Failed to remove segment") .addData("segment", segment) .emit(); + + updateStatusAndResolveWaitingFutures(new SegmentChangeRequestDrop(segment), Status.failed(e.getMessage())); } finally { callback.execute(); @@ -407,6 +471,107 @@ public Collection getPendingDeleteSnapshot() return ImmutableList.copyOf(segmentsToDelete); } + public ListenableFuture>> processBatch(List changeRequests) + { + boolean isAnyRequestDone = false; + + Map> statuses = new HashMap<>(changeRequests.size()); + + for (DataSegmentChangeRequest cr : changeRequests) { + AtomicReference status = processRequest(cr); + if (status.get().getState() != Status.STATE.PENDING) { + isAnyRequestDone = true; + } + statuses.put(cr, status); + } + + CustomSettableFuture future = new CustomSettableFuture(waitingFutures, statuses); + + if (isAnyRequestDone) { + future.resolve(); + } else { + synchronized (waitingFutures) { + waitingFutures.add(future); + } + } + + return future; + } + + private AtomicReference processRequest(DataSegmentChangeRequest changeRequest) + { + synchronized (requestStatusesLock) { + if (requestStatuses.getIfPresent(changeRequest) == null) { + if (changeRequest instanceof SegmentChangeRequestLoad) { + requestStatuses.put(changeRequest, new AtomicReference<>(Status.PENDING)); + exec.submit( + () -> SegmentLoadDropHandler.this.addSegment( + ((SegmentChangeRequestLoad) changeRequest).getSegment(), + DataSegmentChangeCallback.NOOP + ) + ); + } else if (changeRequest instanceof SegmentChangeRequestDrop) { + requestStatuses.put(changeRequest, new AtomicReference<>(Status.PENDING)); + SegmentLoadDropHandler.this.removeSegment( + ((SegmentChangeRequestDrop) changeRequest).getSegment(), + DataSegmentChangeCallback.NOOP, + true + ); + } else { + throw new IAE("Not a segment load/drop request[%s].", changeRequest.getClass().getName()); + } + } + return requestStatuses.getIfPresent(changeRequest); + } + } + + private void updateStatusAndResolveWaitingFutures(DataSegmentChangeRequest chanteRequest, Status result) + { + if (result == null) { + result = Status.failed("Unknown reason. Check server logs."); + } + synchronized (requestStatusesLock) { + AtomicReference statusRef = requestStatuses.getIfPresent(chanteRequest); + if (statusRef != null) { + statusRef.set(result); + } + } + + LinkedHashSet waitingFuturesCopy = new LinkedHashSet<>(); + synchronized (waitingFutures) { + waitingFuturesCopy.addAll(waitingFutures); + waitingFutures.clear(); + } + for (CustomSettableFuture future : waitingFuturesCopy) { + future.resolve(); + } + } + + public DataSegmentChangeHandler getSyncLoadingHanlder() + { + return new DataSegmentChangeHandler() + { + @Override + public void addSegment(DataSegment segment, DataSegmentChangeCallback callback) + { + SegmentLoadDropHandler.this.addSegment(segment, callback); + } + + @Override + public void removeSegment(DataSegment segment, DataSegmentChangeCallback callback) + { + try { + SegmentLoadDropHandler.this.removeSegment(segment, callback, true); + } + catch (Exception ex) { + log.makeAlert(ex, "Failed to remove segment") + .addData("segment", segment) + .emit(); + } + } + }; + } + private static class BackgroundSegmentAnnouncer implements AutoCloseable { private static final EmittingLogger log = new EmittingLogger(BackgroundSegmentAnnouncer.class); @@ -537,4 +702,96 @@ public void close() } } } + + // Future with cancel() implementation to remove it from "waitingFutures" list + private static class CustomSettableFuture extends AbstractFuture>> + { + private final LinkedHashSet waitingFutures; + private final Map> statusRefs; + + private CustomSettableFuture( + LinkedHashSet waitingFutures, + Map> statusRefs + ) + { + this.waitingFutures = waitingFutures; + this.statusRefs = statusRefs; + } + + public void resolve() + { + List> result = new ArrayList<>(statusRefs.size()); + statusRefs.forEach( + (request, statusRef) -> result.add(new Pair(request, statusRef.get())) + ); + + super.set(result); + } + + @Override + public boolean setException(Throwable throwable) + { + return super.setException(throwable); + } + + @Override + public boolean cancel(boolean interruptIfRunning) + { + synchronized (waitingFutures) { + waitingFutures.remove(this); + } + return true; + } + } + + public static class Status + { + public enum STATE + { + SUCCESS, FAILED, PENDING + } + + private final STATE state; + private final String failureCause; + + public final static Status SUCCESS = new Status(STATE.SUCCESS, null); + public final static Status PENDING = new Status(STATE.PENDING, null); + + Status( + @JsonProperty("state") STATE state, + @JsonProperty("failureCause") String failureCause + ) + { + Preconditions.checkNotNull(state, "state must be non-null"); + this.state = state; + this.failureCause = failureCause; + } + + public static Status failed(String cause) + { + return new Status(STATE.FAILED, cause); + } + + @JsonProperty + public STATE getState() + { + return state; + } + + @JsonProperty + public String getFailureCause() + { + return failureCause; + } + + @Override + public String toString() + { + return "Status{" + + "state=" + state + + ", failureCause='" + failureCause + '\'' + + '}'; + } + } } + diff --git a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java index de07f5f19637..65854945af15 100644 --- a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java +++ b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java @@ -26,7 +26,6 @@ import io.druid.concurrent.Execs; import io.druid.java.util.common.lifecycle.LifecycleStart; import io.druid.java.util.common.lifecycle.LifecycleStop; -import io.druid.segment.loading.SegmentLoaderConfig; import io.druid.server.initialization.ZkPathsConfig; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; @@ -45,10 +44,9 @@ public class ZkCoordinator private final Object lock = new Object(); - private final SimpleDataSegmentChangeHandler dataSegmentChangeHandler; + private final DataSegmentChangeHandler dataSegmentChangeHandler; private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPaths; - private final SegmentLoaderConfig config; private final DruidServerMetadata me; private final CuratorFramework curator; @@ -57,18 +55,16 @@ public class ZkCoordinator @Inject public ZkCoordinator( - SimpleDataSegmentChangeHandler dataSegmentChangeHandler, + SegmentLoadDropHandler loadDropHandler, ObjectMapper jsonMapper, - SegmentLoaderConfig config, ZkPathsConfig zkPaths, DruidServerMetadata me, CuratorFramework curator ) { - this.dataSegmentChangeHandler = dataSegmentChangeHandler; + this.dataSegmentChangeHandler = loadDropHandler.getSyncLoadingHanlder(); this.jsonMapper = jsonMapper; this.zkPaths = zkPaths; - this.config = config; this.me = me; this.curator = curator; } @@ -92,7 +88,7 @@ public void start() throws IOException loadQueueLocation, true, true, - Execs.multiThreaded(config.getNumLoadingThreads(), "ZkCoordinator-%s") + Execs.singleThreaded("ZkCoordinator") ); try { diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java index 95d3d0cd1a71..fd51086ee11c 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinator.java @@ -735,10 +735,9 @@ public ImmutableDruidServer apply(DruidServer input) final DruidCluster cluster = new DruidCluster(); for (ImmutableDruidServer server : servers) { if (!loadManagementPeons.containsKey(server.getName())) { - String basePath = ZKPaths.makePath(zkPaths.getLoadQueuePath(), server.getName()); - LoadQueuePeon loadQueuePeon = taskMaster.giveMePeon(basePath); + LoadQueuePeon loadQueuePeon = taskMaster.giveMePeon(server); loadQueuePeon.start(); - log.info("Creating LoadQueuePeon for server[%s] at path[%s]", server.getName(), basePath); + log.info("Created LoadQueuePeon for server[%s].", server.getName()); loadManagementPeons.put(server.getName(), loadQueuePeon); } diff --git a/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorConfig.java b/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorConfig.java index 1271af0ec751..ba834620013d 100644 --- a/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorConfig.java +++ b/server/src/main/java/io/druid/server/coordinator/DruidCoordinatorConfig.java @@ -86,4 +86,28 @@ public Duration getLoadQueuePeonRepeatDelay() { return Duration.millis(50); } + + @Config("druid.coordinator.loadqueuepeon.type") + public String getLoadQueuePeonType() + { + return "curator"; + } + + @Config("druid.coordinator.loadqueuepeon.http.repeatDelay") + public Duration getHttpLoadQueuePeonRepeatDelay() + { + return Duration.millis(60000); + } + + @Config("druid.coordinator.loadqueuepeon.http.hostTimeout") + public Duration getHttpLoadQueuePeonHostTimeout() + { + return Duration.millis(300000); + } + + @Config("druid.coordinator.loadqueuepeon.http.batchSize") + public int getHttpLoadQueuePeonBatchSize() + { + return 1; + } } diff --git a/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java b/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java index be7466003576..dff43824dcc3 100644 --- a/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java +++ b/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java @@ -19,24 +19,67 @@ package io.druid.server.coordinator; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; import com.google.common.collect.Lists; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.metamx.common.RE; +import com.metamx.common.StringUtils; +import com.metamx.emitter.EmittingLogger; +import com.metamx.http.client.HttpClient; +import com.metamx.http.client.Request; +import com.metamx.http.client.io.AppendableByteArrayInputStream; +import com.metamx.http.client.response.ClientResponse; +import com.metamx.http.client.response.InputStreamResponseHandler; +import io.druid.java.util.common.ISE; +import io.druid.java.util.common.Pair; +import io.druid.java.util.common.concurrent.ScheduledExecutors; +import io.druid.server.coordination.DataSegmentChangeCallback; +import io.druid.server.coordination.DataSegmentChangeHandler; import io.druid.server.coordination.DataSegmentChangeRequest; import io.druid.server.coordination.SegmentChangeRequestDrop; import io.druid.server.coordination.SegmentChangeRequestLoad; +import io.druid.server.coordination.SegmentLoadDropHandler; import io.druid.timeline.DataSegment; - -import java.util.Collection; +import org.jboss.netty.handler.codec.http.HttpHeaders; +import org.jboss.netty.handler.codec.http.HttpMethod; +import org.jboss.netty.handler.codec.http.HttpResponse; +import org.joda.time.Duration; + +import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.MediaType; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; /** */ public class HttpLoadQueuePeon extends LoadQueuePeon { + public static final TypeReference REQUEST_ENTITY_TYPE_REF = new TypeReference>() {}; + public static final TypeReference RESPONSE_ENTITY_TYPE_REF = new TypeReference>>() {}; + + private static final EmittingLogger log = new EmittingLogger(HttpLoadQueuePeon.class); + private static final int DROP = 0; private static final int LOAD = 1; @@ -53,23 +96,321 @@ public class HttpLoadQueuePeon extends LoadQueuePeon DruidCoordinator.SEGMENT_COMPARATOR ); - private void mainLoop() + private final ScheduledExecutorService processingExecutor; + + private volatile boolean stopped = false; + + private final Object lock = new Object(); + + private final DruidCoordinatorConfig config; + + private final ObjectMapper jsonMapper; + private final HttpClient httpClient; + private final URL changeRequestURL; + private final String serverId; + + private final AtomicBoolean mainLoopInProgress = new AtomicBoolean(false); + private final ExecutorService callBackExecutor; + + public HttpLoadQueuePeon( + String baseUrl, + ObjectMapper jsonMapper, + HttpClient httpClient, + DruidCoordinatorConfig config, + ScheduledExecutorService processingExecutor, + ExecutorService callBackExecutor + ) + { + this.jsonMapper = jsonMapper; + this.httpClient = httpClient; + this.config = config; + this.processingExecutor = processingExecutor; + this.callBackExecutor = callBackExecutor; + + this.serverId = baseUrl; + try { + this.changeRequestURL = new URL( + new URL(baseUrl), + StringUtils.safeFormat( + "druid-internal/v1/segments/changeRequests?timeout=%d", + config.getHttpLoadQueuePeonHostTimeout().getMillis() + ) + ); + } + catch (MalformedURLException ex) { + throw Throwables.propagate(ex); + } + } + + private void doSegmentManagement() { - //check status of each pending and update + if (stopped || !mainLoopInProgress.compareAndSet(false, true)) { + log.debug("[%s]Ignoring tick. Either in-progress already or stopped.", serverId); + return; + } + + int batchSize = config.getHttpLoadQueuePeonBatchSize(); + + List newRequests = new ArrayList<>(batchSize); + + synchronized (lock) { + Iterator> iter = segmentsToDrop.entrySet().iterator(); + while (batchSize-- > 0 && iter.hasNext()) { + Map.Entry entry = iter.next(); + newRequests.add(entry.getValue().getChangeRequest()); + entry.getValue().scheduleCancellationOnTimeout(); + } + + iter = segmentsToLoad.entrySet().iterator(); + while (batchSize-- > 0 && iter.hasNext()) { + Map.Entry entry = iter.next(); + newRequests.add(entry.getValue().getChangeRequest()); + entry.getValue().scheduleCancellationOnTimeout(); + } + } - //check size of pending list and send request for (MAX - pendingList.size) + if (newRequests.size() > 0) { + try { + log.debug("Sending [%d] load/drop requests to Server[%s].", newRequests.size(), serverId); + BytesAccumulatingResponseHandler responseHandler = new BytesAccumulatingResponseHandler(); + ListenableFuture future = httpClient.go( + new Request(HttpMethod.POST, changeRequestURL) + .addHeader(HttpHeaders.Names.ACCEPT, MediaType.APPLICATION_JSON) + .addHeader(HttpHeaders.Names.CONTENT_TYPE, MediaType.APPLICATION_JSON) + .setContent(jsonMapper.writerWithType(REQUEST_ENTITY_TYPE_REF).writeValueAsBytes(newRequests)), + responseHandler, + new Duration(config.getHttpLoadQueuePeonHostTimeout().getMillis() + 5000) + ); + + Futures.addCallback( + future, + new FutureCallback() + { + @Override + public void onSuccess(InputStream result) + { + boolean scheduleNextRunImmediately = true; + + if (responseHandler.status == HttpServletResponse.SC_NO_CONTENT) { + log.debug("Received NO CONTENT reseponse from [%s]", serverId); + } else if (HttpServletResponse.SC_OK == responseHandler.status) { + try { + List> statuses = jsonMapper.readValue( + result, RESPONSE_ENTITY_TYPE_REF + ); + + synchronized (lock) { + if (stopped) { + return; + } + + for (Pair e : statuses) { + switch (e.rhs.getState()) { + case SUCCESS: + case FAILED: + e.lhs.go( + new DataSegmentChangeHandler() + { + @Override + public void addSegment(DataSegment segment, DataSegmentChangeCallback callback) + { + SegmentHolder holder = segmentsToLoad.remove(segment); + if (holder == null) { + return; + } + + if (e.rhs.getState() + == SegmentLoadDropHandler.Status.STATE.FAILED) { + holder.requestCompleted( + true, + StringUtils.safeFormat( + "Server[%s] Failed segment[%s] LOAD request with cause [%s].", + serverId, + segment.getIdentifier(), + e.rhs.getFailureCause() + ) + ); + } else { + holder.requestCompleted(false, null); + } + + } + + @Override + public void removeSegment(DataSegment segment, DataSegmentChangeCallback callback) + { + SegmentHolder holder = segmentsToDrop.remove(segment); + if (holder == null) { + return; + } + + if (e.rhs.getState() + == SegmentLoadDropHandler.Status.STATE.FAILED) { + holder.requestCompleted( + true, + StringUtils.safeFormat( + "Server[%s] Failed segment[%s] DROP request with cause [%s].", + serverId, + segment.getIdentifier(), + e.rhs.getFailureCause() + ) + ); + } else { + holder.requestCompleted(false, null); + } + } + }, null + ); + break; + case PENDING: + log.debug("[%s]Segment request [%s] is pending.", serverId, e.lhs); + break; + default: + scheduleNextRunImmediately = false; + log.error("WTF! Server[%s] returned unknown state in status[%s].", serverId, e.rhs); + } + } + } + } + catch (Exception ex) { + scheduleNextRunImmediately = false; + onFailure(ex); + } + finally { + mainLoopInProgress.set(false); + } + } else { + scheduleNextRunImmediately = false; + onFailure(new RE("Unexpected Response Status.")); + } + + if (scheduleNextRunImmediately) { + processingExecutor.execute(HttpLoadQueuePeon.this::doSegmentManagement); + } + } + + @Override + public void onFailure(Throwable t) + { + try { + log.info( + "Request[%s] Failed with code[%s] and status[%s]. Reason[%s]. Error [%s]", + changeRequestURL, + responseHandler.status, + responseHandler.description, + t.getMessage() + ); + log.error( + t, + "Request[%s] Failed with code[%s] and status[%s]. Reason[%s].", + changeRequestURL, + responseHandler.status, + responseHandler.description + ); + } + finally { + mainLoopInProgress.set(false); + } + } + }, + processingExecutor + ); + } + catch (Throwable th) { + mainLoopInProgress.set(false); + } + } else { + log.debug("[%s] No segments to load/drop."); + mainLoopInProgress.set(false); + } } @Override public void start() { + synchronized (lock) { + if (stopped) { + throw new ISE("Can't start."); + } + ScheduledExecutors.scheduleAtFixedRate( + processingExecutor, + new Duration(config.getHttpLoadQueuePeonRepeatDelay()), + new Callable() + { + @Override + public ScheduledExecutors.Signal call() + { + if (!stopped) { + doSegmentManagement(); + } + + if (stopped) { + return ScheduledExecutors.Signal.STOP; + } else { + return ScheduledExecutors.Signal.REPEAT; + } + } + } + ); + } } @Override public void stop() { + synchronized (lock) { + if (stopped) { + return; + } + + stopped = true; + + for (SegmentHolder holder : segmentsToDrop.values()) { + holder.requestCompleted(false, null); + } + + for (SegmentHolder holder : segmentsToLoad.values()) { + holder.requestCompleted(false, null); + } + segmentsToDrop.clear(); + segmentsToLoad.clear(); + queuedSize.set(0L); + failedAssignCount.set(0); + } + } + + @Override + public void loadSegment(DataSegment segment, LoadPeonCallback callback) + { + synchronized (lock) { + SegmentHolder holder = segmentsToLoad.get(segment); + + if (holder == null) { + log.info("Server[%s] to load segment[%s] queued.", serverId, segment.getIdentifier()); + segmentsToLoad.put(segment, new SegmentHolder(segment, LOAD, callback)); + processingExecutor.execute(this::doSegmentManagement); + } else { + holder.addCallback(callback); + } + } + } + + @Override + public void dropSegment(DataSegment segment, LoadPeonCallback callback) + { + synchronized (lock) { + SegmentHolder holder = segmentsToDrop.get(segment); + + if (holder == null) { + log.info("Server[%s] to drop segment[%s] queued.", serverId, segment.getIdentifier()); + segmentsToDrop.put(segment, new SegmentHolder(segment, DROP, callback)); + processingExecutor.execute(this::doSegmentManagement); + } else { + holder.addCallback(callback); + } + } } @Override @@ -120,19 +461,7 @@ public Set getSegmentsMarkedToDrop() return segmentsMarkedToDrop; } - @Override - public void loadSegment(DataSegment segment, LoadPeonCallback callback) - { - - } - - @Override - public void dropSegment(DataSegment segment, LoadPeonCallback callback) - { - - } - - private static class SegmentHolder + private class SegmentHolder { private final DataSegment segment; private final DataSegmentChangeRequest changeRequest; @@ -142,7 +471,7 @@ private static class SegmentHolder private SegmentHolder( DataSegment segment, int type, - Collection callbacks + LoadPeonCallback callback ) { this.segment = segment; @@ -150,7 +479,14 @@ private SegmentHolder( this.changeRequest = (type == LOAD) ? new SegmentChangeRequestLoad(segment) : new SegmentChangeRequestDrop(segment); - this.callbacks.addAll(callbacks); + + if (callback != null) { + this.callbacks.add(callback); + } + + if (type == LOAD) { + queuedSize.addAndGet(segment.getSize()); + } } public DataSegment getSegment() @@ -163,46 +499,105 @@ public int getType() return type; } - public String getSegmentIdentifier() + public void addCallback(LoadPeonCallback newCallback) { - return segment.getIdentifier(); + synchronized (callbacks) { + if (newCallback != null) { + callbacks.add(newCallback); + } + } } - public long getSegmentSize() + public DataSegmentChangeRequest getChangeRequest() { - return segment.getSize(); + return changeRequest; } - public void addCallbacks(Collection newCallbacks) - { - synchronized (callbacks) { - callbacks.addAll(newCallbacks); - } - } + AtomicReference cancellationFutureRef = new AtomicReference<>(); - public void addCallback(LoadPeonCallback newCallback) + public void scheduleCancellationOnTimeout() { - synchronized (callbacks) { - callbacks.add(newCallback); - } + cancellationFutureRef.updateAndGet( + (curr) -> { + if (curr == null) { + return processingExecutor.schedule( + () -> { + synchronized (lock) { + if (type == LOAD) { + segmentsToLoad.remove(segment); + } else { + segmentsToDrop.remove(segment); + } + requestCompleted( + true, + StringUtils.safeFormat( + "Server[%s] failed to complete request[%s] within timeout[%s millis].", + serverId, + config.getLoadTimeoutDelay().getMillis() + ) + ); + } + }, + config.getLoadTimeoutDelay().getMillis(), + TimeUnit.MILLISECONDS + ); + } else { + return curr; + } + } + ); } - public List getCallbacks() + public void requestCompleted(boolean failed, String failMsg) { - synchronized (callbacks) { - return callbacks; + synchronized (lock) { + Future cancellationFuture = cancellationFutureRef.get(); + + if (cancellationFuture != null && cancellationFuture.isDone()) { + return; + } + + if (cancellationFuture != null) { + cancellationFuture.cancel(true); + } + + if (failed) { + log.error(failMsg); + failedAssignCount.getAndIncrement(); + } + + if (type == LOAD) { + queuedSize.addAndGet(-segment.getSize()); + } + + callBackExecutor.execute(() -> { + for (LoadPeonCallback callback : callbacks) { + if (callback != null) { + callback.execute(); + } + } + }); } } - public DataSegmentChangeRequest getChangeRequest() + @Override + public String toString() { - return changeRequest; + return changeRequest.toString(); } + } + + private static class BytesAccumulatingResponseHandler extends InputStreamResponseHandler + { + private int status; + private String description; @Override - public String toString() + public ClientResponse handleResponse(HttpResponse response) { - return changeRequest.toString(); + status = response.getStatus().getCode(); + description = response.getStatus().getReasonPhrase(); + return ClientResponse.unfinished(super.handleResponse(response).getObj()); } } } diff --git a/server/src/main/java/io/druid/server/coordinator/LoadQueuePeon.java b/server/src/main/java/io/druid/server/coordinator/LoadQueuePeon.java index b46ff5844608..658be22e4f34 100644 --- a/server/src/main/java/io/druid/server/coordinator/LoadQueuePeon.java +++ b/server/src/main/java/io/druid/server/coordinator/LoadQueuePeon.java @@ -24,7 +24,10 @@ import java.util.Set; /** + * This interface exists only to support configurable load queue management via curator or http. Once HttpLoadQueuePeon + * has been verified enough in production, CuratorLoadQueuePeon and this interface would be removed. */ +@Deprecated public abstract class LoadQueuePeon { public abstract void start(); diff --git a/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java b/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java index 367198060735..c24a98a2ff2f 100644 --- a/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java +++ b/server/src/main/java/io/druid/server/coordinator/LoadQueueTaskMaster.java @@ -21,7 +21,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; +import com.metamx.http.client.HttpClient; +import io.druid.client.ImmutableDruidServer; +import io.druid.guice.annotations.Global; +import io.druid.guice.annotations.Json; +import io.druid.server.initialization.ZkPathsConfig; import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.utils.ZKPaths; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; @@ -36,14 +42,18 @@ public class LoadQueueTaskMaster private final ScheduledExecutorService peonExec; private final ExecutorService callbackExec; private final DruidCoordinatorConfig config; + private final HttpClient httpClient; + private final ZkPathsConfig zkPaths; @Inject public LoadQueueTaskMaster( CuratorFramework curator, - ObjectMapper jsonMapper, + @Json ObjectMapper jsonMapper, ScheduledExecutorService peonExec, ExecutorService callbackExec, - DruidCoordinatorConfig config + DruidCoordinatorConfig config, + @Global HttpClient httpClient, + ZkPathsConfig zkPaths ) { this.curator = curator; @@ -51,10 +61,23 @@ public LoadQueueTaskMaster( this.peonExec = peonExec; this.callbackExec = callbackExec; this.config = config; + this.httpClient = httpClient; + this.zkPaths = zkPaths; } - public LoadQueuePeon giveMePeon(String basePath) + public LoadQueuePeon giveMePeon(ImmutableDruidServer server) { - return new CuratorLoadQueuePeon(curator, basePath, jsonMapper, peonExec, callbackExec, config); + if ("http".equalsIgnoreCase(config.getLoadQueuePeonType())) { + return new HttpLoadQueuePeon(server.getURL(), jsonMapper, httpClient, config, peonExec, callbackExec); + } else { + return new CuratorLoadQueuePeon( + curator, + ZKPaths.makePath(zkPaths.getLoadQueuePath(), server.getName()), + jsonMapper, + peonExec, + callbackExec, + config + ); + } } } diff --git a/server/src/main/java/io/druid/server/http/SegmentListerResource.java b/server/src/main/java/io/druid/server/http/SegmentListerResource.java index c24ec4c60a1c..9f545dcc73ec 100644 --- a/server/src/main/java/io/druid/server/http/SegmentListerResource.java +++ b/server/src/main/java/io/druid/server/http/SegmentListerResource.java @@ -29,11 +29,14 @@ import com.sun.jersey.spi.container.ResourceFilters; import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Smile; +import io.druid.java.util.common.Pair; import io.druid.server.coordination.BatchDataSegmentAnnouncer; +import io.druid.server.coordination.DataSegmentChangeRequest; import io.druid.server.coordination.SegmentChangeRequestHistory; import io.druid.server.coordination.SegmentChangeRequestsSnapshot; +import io.druid.server.coordination.SegmentLoadDropHandler; +import io.druid.server.coordinator.HttpLoadQueuePeon; import io.druid.server.http.security.StateResourceFilter; -import io.druid.server.security.AuthConfig; import javax.annotation.Nullable; import javax.servlet.AsyncContext; @@ -43,14 +46,17 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.io.IOException; +import java.util.List; /** + * Endpoints exposed here are to be used only for druid internal management of segments by Coordinators, Brokers etc. */ @Path("/druid-internal/v1/segments/") @ResourceFilters(StateResourceFilter.class) @@ -60,21 +66,21 @@ public class SegmentListerResource protected final ObjectMapper jsonMapper; protected final ObjectMapper smileMapper; - protected final AuthConfig authConfig; private final BatchDataSegmentAnnouncer announcer; + private final SegmentLoadDropHandler loadDropRequestHandler; @Inject public SegmentListerResource( @Json ObjectMapper jsonMapper, @Smile ObjectMapper smileMapper, - AuthConfig authConfig, - @Nullable BatchDataSegmentAnnouncer announcer + @Nullable BatchDataSegmentAnnouncer announcer, + @Nullable SegmentLoadDropHandler loadDropRequestHandler ) { this.jsonMapper = jsonMapper; this.smileMapper = smileMapper; - this.authConfig = authConfig; this.announcer = announcer; + this.loadDropRequestHandler = loadDropRequestHandler; } /** @@ -203,6 +209,116 @@ public void onFailure(Throwable th) asyncContext.setTimeout(timeout); } + /** + * This endpoint is used by HttpLoadQueuePeon to assign segment load/drop requests batch. This endpoint makes the + * client wait till one of the following events occur. Note that this is implemented using async IO so no jetty + * threads are held while in wait. + * + * (1) Given timeout elapses. + * (2) Some load/drop request completed. + * + * It returns a map of "load/drop request -> SUCCESS/FAILED/PENDING status" for each request in the batch. + */ + @POST + @Path("/changeRequests") + @Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE}) + @Consumes({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE}) + public void applyDataSegmentChangeRequests( + @QueryParam("timeout") long timeout, + List changeRequestList, + @Context final HttpServletRequest req + ) throws IOException + { + if (loadDropRequestHandler == null) { + sendErrorResponse(req, HttpServletResponse.SC_NOT_FOUND, "load/drop handler is not available."); + return; + } + + if (timeout <= 0) { + sendErrorResponse(req, HttpServletResponse.SC_BAD_REQUEST, "timeout must be positive."); + return; + } + + if (changeRequestList == null || changeRequestList.isEmpty()) { + sendErrorResponse(req, HttpServletResponse.SC_BAD_REQUEST, "No change requests provided."); + return; + } + + final ResponseContext context = createContext(req.getHeader("Accept")); + final ListenableFuture>> future = loadDropRequestHandler + .processBatch(changeRequestList); + + final AsyncContext asyncContext = req.startAsync(); + + asyncContext.addListener( + new AsyncListener() + { + @Override + public void onComplete(AsyncEvent event) throws IOException + { + } + + @Override + public void onTimeout(AsyncEvent event) throws IOException + { + + // HTTP 204 NO_CONTENT is sent to the client. + future.cancel(true); + event.getAsyncContext().complete(); + } + + @Override + public void onError(AsyncEvent event) throws IOException + { + } + + @Override + public void onStartAsync(AsyncEvent event) throws IOException + { + } + } + ); + + Futures.addCallback( + future, + new FutureCallback>>() + { + @Override + public void onSuccess(List> result) + { + try { + HttpServletResponse response = (HttpServletResponse) asyncContext.getResponse(); + response.setStatus(HttpServletResponse.SC_OK); + context.inputMapper.writerWithType(HttpLoadQueuePeon.RESPONSE_ENTITY_TYPE_REF).writeValue(asyncContext.getResponse().getOutputStream(), result); + asyncContext.complete(); + } + catch (Exception ex) { + log.debug(ex, "Request timed out or closed already."); + } + } + + @Override + public void onFailure(Throwable th) + { + try { + HttpServletResponse response = (HttpServletResponse) asyncContext.getResponse(); + if (th instanceof IllegalArgumentException) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, th.getMessage()); + } else { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, th.getMessage()); + } + asyncContext.complete(); + } + catch (Exception ex) { + log.debug(ex, "Request timed out or closed already."); + } + } + } + ); + + asyncContext.setTimeout(timeout); + } + private void sendErrorResponse(HttpServletRequest req, int code, String error) throws IOException { AsyncContext asyncContext = req.startAsync(); diff --git a/server/src/main/java/io/druid/server/metrics/HistoricalMetricsMonitor.java b/server/src/main/java/io/druid/server/metrics/HistoricalMetricsMonitor.java index f6bd6e2e6059..3d775984a135 100644 --- a/server/src/main/java/io/druid/server/metrics/HistoricalMetricsMonitor.java +++ b/server/src/main/java/io/druid/server/metrics/HistoricalMetricsMonitor.java @@ -26,7 +26,7 @@ import io.druid.client.DruidServerConfig; import io.druid.query.DruidMetrics; import io.druid.server.SegmentManager; -import io.druid.server.coordination.ZkCoordinator; +import io.druid.server.coordination.SegmentLoadDropHandler; import io.druid.timeline.DataSegment; import it.unimi.dsi.fastutil.objects.Object2LongMap; import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap; @@ -37,18 +37,18 @@ public class HistoricalMetricsMonitor extends AbstractMonitor { private final DruidServerConfig serverConfig; private final SegmentManager segmentManager; - private final ZkCoordinator zkCoordinator; + private final SegmentLoadDropHandler segmentLoadDropMgr; @Inject public HistoricalMetricsMonitor( DruidServerConfig serverConfig, SegmentManager segmentManager, - ZkCoordinator zkCoordinator + SegmentLoadDropHandler segmentLoadDropMgr ) { this.serverConfig = serverConfig; this.segmentManager = segmentManager; - this.zkCoordinator = zkCoordinator; + this.segmentLoadDropMgr = segmentLoadDropMgr; } @Override @@ -58,7 +58,7 @@ public boolean doMonitor(ServiceEmitter emitter) final Object2LongOpenHashMap pendingDeleteSizes = new Object2LongOpenHashMap<>(); - for (DataSegment segment : zkCoordinator.getPendingDeleteSnapshot()) { + for (DataSegment segment : segmentLoadDropMgr.getPendingDeleteSnapshot()) { pendingDeleteSizes.addTo(segment.getDataSource(), segment.getSize()); } diff --git a/server/src/test/java/io/druid/server/coordination/SegmentLoadDropHandlerTest.java b/server/src/test/java/io/druid/server/coordination/SegmentLoadDropHandlerTest.java new file mode 100644 index 000000000000..078152933260 --- /dev/null +++ b/server/src/test/java/io/druid/server/coordination/SegmentLoadDropHandlerTest.java @@ -0,0 +1,483 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.coordination; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.java.util.common.Intervals; +import io.druid.java.util.common.Pair; +import io.druid.java.util.common.concurrent.ScheduledExecutorFactory; +import io.druid.java.util.common.logger.Logger; +import io.druid.segment.IndexIO; +import io.druid.segment.loading.CacheTestSegmentLoader; +import io.druid.segment.loading.SegmentLoaderConfig; +import io.druid.server.SegmentManager; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; +import org.easymock.EasyMock; +import org.joda.time.Interval; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + */ +public class SegmentLoadDropHandlerTest +{ + public static final int COUNT = 50; + + private static final Logger log = new Logger(ZkCoordinatorTest.class); + + private final ObjectMapper jsonMapper = new DefaultObjectMapper(); + private final DruidServerMetadata me = new DruidServerMetadata( + "dummyServer", + "dummyHost", + null, + 0, + ServerType.HISTORICAL, + "normal", + 0 + ); + + private SegmentLoadDropHandler segmentLoadDropHandler; + private DataSegmentAnnouncer announcer; + private File infoDir; + private AtomicInteger announceCount; + private ConcurrentSkipListSet segmentsAnnouncedByMe; + private CacheTestSegmentLoader segmentLoader; + private SegmentManager segmentManager; + private List scheduledRunnable; + + @Before + public void setUp() throws Exception + { + try { + infoDir = new File(File.createTempFile("blah", "blah2").getParent(), "ZkCoordinatorTest"); + infoDir.mkdirs(); + for (File file : infoDir.listFiles()) { + file.delete(); + } + log.info("Creating tmp test files in [%s]", infoDir); + } + catch (IOException e) { + throw new RuntimeException(e); + } + + scheduledRunnable = Lists.newArrayList(); + + segmentLoader = new CacheTestSegmentLoader(); + segmentManager = new SegmentManager(segmentLoader); + final ZkPathsConfig zkPaths = new ZkPathsConfig() + { + @Override + public String getBase() + { + return "/druid"; + } + }; + + segmentsAnnouncedByMe = new ConcurrentSkipListSet<>(); + announceCount = new AtomicInteger(0); + + announcer = new DataSegmentAnnouncer() + { + @Override + public void announceSegment(DataSegment segment) throws IOException + { + segmentsAnnouncedByMe.add(segment); + announceCount.incrementAndGet(); + } + + @Override + public void unannounceSegment(DataSegment segment) throws IOException + { + segmentsAnnouncedByMe.remove(segment); + announceCount.decrementAndGet(); + } + + @Override + public void announceSegments(Iterable segments) throws IOException + { + for (DataSegment segment : segments) { + segmentsAnnouncedByMe.add(segment); + } + announceCount.addAndGet(Iterables.size(segments)); + } + + @Override + public void unannounceSegments(Iterable segments) throws IOException + { + for (DataSegment segment : segments) { + segmentsAnnouncedByMe.remove(segment); + } + announceCount.addAndGet(-Iterables.size(segments)); + } + }; + + segmentLoadDropHandler = new SegmentLoadDropHandler( + jsonMapper, + new SegmentLoaderConfig() + { + @Override + public File getInfoDir() + { + return infoDir; + } + + @Override + public int getNumLoadingThreads() + { + return 5; + } + + @Override + public int getAnnounceIntervalMillis() + { + return 50; + } + + @Override + public int getDropSegmentDelayMillis() + { + return 0; + } + }, + announcer, + EasyMock.createNiceMock(DataSegmentServerAnnouncer.class), + segmentManager, + new ScheduledExecutorFactory() + { + @Override + public ScheduledExecutorService create(int corePoolSize, String nameFormat) + { + /* + Override normal behavoir by adding the runnable to a list so that you can make sure + all the shceduled runnables are executed by explicitly calling run() on each item in the list + */ + return new ScheduledThreadPoolExecutor( + corePoolSize, new ThreadFactoryBuilder().setDaemon(true).setNameFormat(nameFormat).build() + ) + { + @Override + public ScheduledFuture schedule( + Runnable command, long delay, TimeUnit unit + ) + { + scheduledRunnable.add(command); + return null; + } + }; + } + }.create(5, "SegmentLoadDropHandlerTest-[%d]") + ); + } + + /** + * Steps: + * 1. removeSegment() schedules a delete runnable that deletes segment files, + * 2. addSegment() succesfully loads the segment and annouces it + * 3. scheduled delete task executes and realizes it should not delete the segment files. + */ + @Test + public void testSegmentLoading1() throws Exception + { + segmentLoadDropHandler.start(); + + final DataSegment segment = makeSegment("test", "1", Intervals.of("P1d/2011-04-01")); + + segmentLoadDropHandler.removeSegment(segment, DataSegmentChangeCallback.NOOP, true); + + Assert.assertFalse(segmentsAnnouncedByMe.contains(segment)); + + segmentLoadDropHandler.addSegment(segment, DataSegmentChangeCallback.NOOP); + + /* + make sure the scheduled runnable that "deletes" segment files has been executed. + Because another addSegment() call is executed, which removes the segment from segmentsToDelete field in + ZkCoordinator, the scheduled runnable will not actually delete segment files. + */ + for (Runnable runnable : scheduledRunnable) { + runnable.run(); + } + + Assert.assertTrue(segmentsAnnouncedByMe.contains(segment)); + Assert.assertFalse("segment files shouldn't be deleted", segmentLoader.getSegmentsInTrash().contains(segment)); + + segmentLoadDropHandler.stop(); + } + + /** + * Steps: + * 1. addSegment() succesfully loads the segment and annouces it + * 2. removeSegment() unannounces the segment and schedules a delete runnable that deletes segment files + * 3. addSegment() calls loadSegment() and annouces it again + * 4. scheduled delete task executes and realizes it should not delete the segment files. + */ + @Test + public void testSegmentLoading2() throws Exception + { + segmentLoadDropHandler.start(); + + final DataSegment segment = makeSegment("test", "1", Intervals.of("P1d/2011-04-01")); + + segmentLoadDropHandler.addSegment(segment, DataSegmentChangeCallback.NOOP); + + Assert.assertTrue(segmentsAnnouncedByMe.contains(segment)); + + segmentLoadDropHandler.removeSegment(segment, DataSegmentChangeCallback.NOOP, true); + + Assert.assertFalse(segmentsAnnouncedByMe.contains(segment)); + + segmentLoadDropHandler.addSegment(segment, DataSegmentChangeCallback.NOOP); + + /* + make sure the scheduled runnable that "deletes" segment files has been executed. + Because another addSegment() call is executed, which removes the segment from segmentsToDelete field in + ZkCoordinator, the scheduled runnable will not actually delete segment files. + */ + for (Runnable runnable : scheduledRunnable) { + runnable.run(); + } + + Assert.assertTrue(segmentsAnnouncedByMe.contains(segment)); + Assert.assertFalse("segment files shouldn't be deleted", segmentLoader.getSegmentsInTrash().contains(segment)); + + segmentLoadDropHandler.stop(); + } + + @Test + public void testLoadCache() throws Exception + { + List segments = Lists.newLinkedList(); + for (int i = 0; i < COUNT; ++i) { + segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-01"))); + segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-02"))); + segments.add(makeSegment("test" + i, "2", Intervals.of("P1d/2011-04-02"))); + segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-03"))); + segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-04"))); + segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-05"))); + segments.add(makeSegment("test" + i, "2", Intervals.of("PT1h/2011-04-04T01"))); + segments.add(makeSegment("test" + i, "2", Intervals.of("PT1h/2011-04-04T02"))); + segments.add(makeSegment("test" + i, "2", Intervals.of("PT1h/2011-04-04T03"))); + segments.add(makeSegment("test" + i, "2", Intervals.of("PT1h/2011-04-04T05"))); + segments.add(makeSegment("test" + i, "2", Intervals.of("PT1h/2011-04-04T06"))); + segments.add(makeSegment("test_two" + i, "1", Intervals.of("P1d/2011-04-01"))); + segments.add(makeSegment("test_two" + i, "1", Intervals.of("P1d/2011-04-02"))); + } + Collections.sort(segments); + + for (DataSegment segment : segments) { + writeSegmentToCache(segment); + } + + checkCache(segments); + Assert.assertTrue(segmentManager.getDataSourceCounts().isEmpty()); + segmentLoadDropHandler.start(); + Assert.assertTrue(!segmentManager.getDataSourceCounts().isEmpty()); + for (int i = 0; i < COUNT; ++i) { + Assert.assertEquals(11L, segmentManager.getDataSourceCounts().get("test" + i).longValue()); + Assert.assertEquals(2L, segmentManager.getDataSourceCounts().get("test_two" + i).longValue()); + } + Assert.assertEquals(13 * COUNT, announceCount.get()); + segmentLoadDropHandler.stop(); + + for (DataSegment segment : segments) { + deleteSegmentFromCache(segment); + } + + Assert.assertEquals(0, infoDir.listFiles().length); + Assert.assertTrue(infoDir.delete()); + } + + private DataSegment makeSegment(String dataSource, String version, Interval interval) + { + return new DataSegment( + dataSource, + interval, + version, + ImmutableMap.of("version", version, "interval", interval, "cacheDir", infoDir), + Arrays.asList("dim1", "dim2", "dim3"), + Arrays.asList("metric1", "metric2"), + NoneShardSpec.instance(), + IndexIO.CURRENT_VERSION_ID, + 123L + ); + } + + private void writeSegmentToCache(final DataSegment segment) throws IOException + { + if (!infoDir.exists()) { + infoDir.mkdir(); + } + + File segmentInfoCacheFile = new File( + infoDir, + segment.getIdentifier() + ); + try { + jsonMapper.writeValue(segmentInfoCacheFile, segment); + } + catch (IOException e) { + throw new RuntimeException(e); + } + + Assert.assertTrue(segmentInfoCacheFile.exists()); + } + + private void deleteSegmentFromCache(final DataSegment segment) throws IOException + { + File segmentInfoCacheFile = new File( + infoDir, + segment.getIdentifier() + ); + if (segmentInfoCacheFile.exists()) { + segmentInfoCacheFile.delete(); + } + + Assert.assertTrue(!segmentInfoCacheFile.exists()); + } + + private void checkCache(List segments) throws IOException + { + Assert.assertTrue(infoDir.exists()); + File[] files = infoDir.listFiles(); + + List sortedFiles = Lists.newArrayList(files); + Collections.sort(sortedFiles); + + Assert.assertEquals(segments.size(), sortedFiles.size()); + for (int i = 0; i < sortedFiles.size(); i++) { + DataSegment segment = jsonMapper.readValue(sortedFiles.get(i), DataSegment.class); + Assert.assertEquals(segments.get(i), segment); + } + } + + @Test + public void testStartStop() throws Exception + { + SegmentLoadDropHandler handler = new SegmentLoadDropHandler( + jsonMapper, + new SegmentLoaderConfig() + { + @Override + public File getInfoDir() + { + return infoDir; + } + + @Override + public int getNumLoadingThreads() + { + return 5; + } + + @Override + public int getAnnounceIntervalMillis() + { + return 50; + } + }, + announcer, EasyMock.createNiceMock(DataSegmentServerAnnouncer.class), segmentManager + ); + + List segments = Lists.newLinkedList(); + for (int i = 0; i < COUNT; ++i) { + segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-01"))); + segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-02"))); + segments.add(makeSegment("test" + i, "2", Intervals.of("P1d/2011-04-02"))); + segments.add(makeSegment("test_two" + i, "1", Intervals.of("P1d/2011-04-01"))); + segments.add(makeSegment("test_two" + i, "1", Intervals.of("P1d/2011-04-02"))); + } + Collections.sort(segments); + + for (DataSegment segment : segments) { + writeSegmentToCache(segment); + } + + checkCache(segments); + Assert.assertTrue(segmentManager.getDataSourceCounts().isEmpty()); + + handler.start(); + Assert.assertTrue(!segmentManager.getDataSourceCounts().isEmpty()); + for (int i = 0; i < COUNT; ++i) { + Assert.assertEquals(3L, segmentManager.getDataSourceCounts().get("test" + i).longValue()); + Assert.assertEquals(2L, segmentManager.getDataSourceCounts().get("test_two" + i).longValue()); + } + Assert.assertEquals(5 * COUNT, announceCount.get()); + handler.stop(); + + for (DataSegment segment : segments) { + deleteSegmentFromCache(segment); + } + + Assert.assertEquals(0, infoDir.listFiles().length); + Assert.assertTrue(infoDir.delete()); + } + + @Test(timeout = 1000L) + public void testProcessBatch() throws Exception + { + segmentLoadDropHandler.start(); + + DataSegment segment1 = makeSegment("batchtest1", "1", Intervals.of("P1d/2011-04-01")); + DataSegment segment2 = makeSegment("batchtest2", "1", Intervals.of("P1d/2011-04-01")); + + List batch = ImmutableList.of( + new SegmentChangeRequestLoad(segment1), + new SegmentChangeRequestDrop(segment2) + ); + + ListenableFuture>> future = segmentLoadDropHandler + .processBatch(batch); + + for (Runnable runnable : scheduledRunnable) { + runnable.run(); + } + + List> result = future.get(); + Assert.assertEquals(SegmentLoadDropHandler.Status.SUCCESS, result.get(0).rhs); + Assert.assertEquals(SegmentLoadDropHandler.Status.PENDING, result.get(1).rhs); + + for (Pair p : segmentLoadDropHandler.processBatch(batch).get()) { + Assert.assertEquals(SegmentLoadDropHandler.Status.SUCCESS, p.rhs); + } + + segmentLoadDropHandler.stop(); + } +} diff --git a/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java index dc14592b820b..36e5b72ba45a 100644 --- a/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java +++ b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java @@ -21,63 +21,29 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.inject.Binder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import io.druid.client.cache.CacheConfig; -import io.druid.client.cache.LocalCacheProvider; -import io.druid.concurrent.Execs; +import com.metamx.emitter.EmittingLogger; import io.druid.curator.CuratorTestBase; -import io.druid.curator.announcement.Announcer; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Intervals; -import io.druid.java.util.common.concurrent.ScheduledExecutorFactory; -import io.druid.java.util.common.concurrent.ScheduledExecutors; -import io.druid.java.util.common.lifecycle.Lifecycle; -import io.druid.java.util.common.logger.Logger; -import io.druid.query.NoopQueryRunnerFactoryConglomerate; import io.druid.segment.IndexIO; -import io.druid.segment.loading.CacheTestSegmentLoader; -import io.druid.segment.loading.SegmentLoaderConfig; -import io.druid.server.SegmentManager; -import io.druid.server.initialization.BatchDataSegmentAnnouncerConfig; import io.druid.server.initialization.ZkPathsConfig; import io.druid.server.metrics.NoopServiceEmitter; import io.druid.timeline.DataSegment; import io.druid.timeline.partition.NoneShardSpec; -import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.utils.ZKPaths; +import org.apache.zookeeper.CreateMode; import org.easymock.EasyMock; -import org.joda.time.Interval; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import java.io.File; -import java.io.IOException; import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.ConcurrentSkipListSet; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.CountDownLatch; /** */ public class ZkCoordinatorTest extends CuratorTestBase { - public static final int COUNT = 50; - - private static final Logger log = new Logger(ZkCoordinatorTest.class); - private final ObjectMapper jsonMapper = new DefaultObjectMapper(); private final DruidServerMetadata me = new DruidServerMetadata( "dummyServer", @@ -88,16 +54,15 @@ public class ZkCoordinatorTest extends CuratorTestBase "normal", 0 ); - + private final ZkPathsConfig zkPaths = new ZkPathsConfig() + { + @Override + public String getBase() + { + return "/druid"; + } + }; private ZkCoordinator zkCoordinator; - private ServerManager serverManager; - private DataSegmentAnnouncer announcer; - private File infoDir; - private AtomicInteger announceCount; - private ConcurrentSkipListSet segmentsAnnouncedByMe; - private CacheTestSegmentLoader segmentLoader; - private SegmentManager segmentManager; - private List scheduledRunnable; @Before public void setUp() throws Exception @@ -105,152 +70,6 @@ public void setUp() throws Exception setupServerAndCurator(); curator.start(); curator.blockUntilConnected(); - try { - infoDir = new File(File.createTempFile("blah", "blah2").getParent(), "ZkCoordinatorTest"); - infoDir.mkdirs(); - for (File file : infoDir.listFiles()) { - file.delete(); - } - log.info("Creating tmp test files in [%s]", infoDir); - } - catch (IOException e) { - throw new RuntimeException(e); - } - - scheduledRunnable = Lists.newArrayList(); - - segmentLoader = new CacheTestSegmentLoader(); - segmentManager = new SegmentManager(segmentLoader); - - serverManager = new ServerManager( - new NoopQueryRunnerFactoryConglomerate(), - new NoopServiceEmitter(), - MoreExecutors.sameThreadExecutor(), - MoreExecutors.sameThreadExecutor(), - new DefaultObjectMapper(), - new LocalCacheProvider().get(), - new CacheConfig(), - segmentManager - ); - - final ZkPathsConfig zkPaths = new ZkPathsConfig() - { - @Override - public String getBase() - { - return "/druid"; - } - }; - - segmentsAnnouncedByMe = new ConcurrentSkipListSet<>(); - announceCount = new AtomicInteger(0); - - announcer = new DataSegmentAnnouncer() - { - private final DataSegmentAnnouncer delegate = new BatchDataSegmentAnnouncer( - me, - new BatchDataSegmentAnnouncerConfig(), - zkPaths, - new Announcer(curator, Execs.singleThreaded("blah")), - jsonMapper - ); - - @Override - public void announceSegment(DataSegment segment) throws IOException - { - segmentsAnnouncedByMe.add(segment); - announceCount.incrementAndGet(); - delegate.announceSegment(segment); - } - - @Override - public void unannounceSegment(DataSegment segment) throws IOException - { - segmentsAnnouncedByMe.remove(segment); - announceCount.decrementAndGet(); - delegate.unannounceSegment(segment); - } - - @Override - public void announceSegments(Iterable segments) throws IOException - { - for (DataSegment segment : segments) { - segmentsAnnouncedByMe.add(segment); - } - announceCount.addAndGet(Iterables.size(segments)); - delegate.announceSegments(segments); - } - - @Override - public void unannounceSegments(Iterable segments) throws IOException - { - for (DataSegment segment : segments) { - segmentsAnnouncedByMe.remove(segment); - } - announceCount.addAndGet(-Iterables.size(segments)); - delegate.unannounceSegments(segments); - } - }; - - zkCoordinator = new ZkCoordinator( - jsonMapper, - new SegmentLoaderConfig() - { - @Override - public File getInfoDir() - { - return infoDir; - } - - @Override - public int getNumLoadingThreads() - { - return 5; - } - - @Override - public int getAnnounceIntervalMillis() - { - return 50; - } - - @Override - public int getDropSegmentDelayMillis() - { - return 0; - } - }, - zkPaths, - me, - announcer, - EasyMock.createNiceMock(DataSegmentServerAnnouncer.class), - curator, - segmentManager, - new ScheduledExecutorFactory() - { - @Override - public ScheduledExecutorService create(int corePoolSize, String nameFormat) - { - /* - Override normal behavoir by adding the runnable to a list so that you can make sure - all the shceduled runnables are executed by explicitly calling run() on each item in the list - */ - return new ScheduledThreadPoolExecutor( - corePoolSize, new ThreadFactoryBuilder().setDaemon(true).setNameFormat(nameFormat).build() - ) - { - @Override - public ScheduledFuture schedule( - Runnable command, long delay, TimeUnit unit - ) - { - scheduledRunnable.add(command); - return null; - } - }; - } - } - ); } @After @@ -259,309 +78,85 @@ public void tearDown() throws Exception tearDownServerAndCurator(); } - /** - * Steps: - * 1. removeSegment() schedules a delete runnable that deletes segment files, - * 2. addSegment() succesfully loads the segment and annouces it - * 3. scheduled delete task executes and realizes it should not delete the segment files. - */ - @Test - public void testSegmentLoading1() throws Exception - { - zkCoordinator.start(); - - final DataSegment segment = makeSegment("test", "1", Intervals.of("P1d/2011-04-01")); - - zkCoordinator.removeSegment(segment, new DataSegmentChangeCallback() - { - @Override - public void execute() - { - // do nothing - } - }); - - Assert.assertFalse(segmentsAnnouncedByMe.contains(segment)); - - zkCoordinator.addSegment(segment, new DataSegmentChangeCallback() - { - @Override - public void execute() - { - // do nothing - } - }); - - /* - make sure the scheduled runnable that "deletes" segment files has been executed. - Because another addSegment() call is executed, which removes the segment from segmentsToDelete field in - ZkCoordinator, the scheduled runnable will not actually delete segment files. - */ - for (Runnable runnable : scheduledRunnable) { - runnable.run(); - } - - Assert.assertTrue(segmentsAnnouncedByMe.contains(segment)); - Assert.assertFalse("segment files shouldn't be deleted", segmentLoader.getSegmentsInTrash().contains(segment)); - - zkCoordinator.stop(); - } - - /** - * Steps: - * 1. addSegment() succesfully loads the segment and annouces it - * 2. removeSegment() unannounces the segment and schedules a delete runnable that deletes segment files - * 3. addSegment() calls loadSegment() and annouces it again - * 4. scheduled delete task executes and realizes it should not delete the segment files. - */ - @Test - public void testSegmentLoading2() throws Exception + @Test(timeout = 5000L) + public void testLoadDrop() throws Exception { - zkCoordinator.start(); - - final DataSegment segment = makeSegment("test", "1", Intervals.of("P1d/2011-04-01")); - - zkCoordinator.addSegment(segment, new DataSegmentChangeCallback() - { - @Override - public void execute() - { - // do nothing - } - }); - - Assert.assertTrue(segmentsAnnouncedByMe.contains(segment)); - - zkCoordinator.removeSegment(segment, new DataSegmentChangeCallback() - { - @Override - public void execute() - { - // do nothing - } - }); - - Assert.assertFalse(segmentsAnnouncedByMe.contains(segment)); - - zkCoordinator.addSegment(segment, new DataSegmentChangeCallback() - { - @Override - public void execute() - { - // do nothing - } - }); - - /* - make sure the scheduled runnable that "deletes" segment files has been executed. - Because another addSegment() call is executed, which removes the segment from segmentsToDelete field in - ZkCoordinator, the scheduled runnable will not actually delete segment files. - */ - for (Runnable runnable : scheduledRunnable) { - runnable.run(); - } - - Assert.assertTrue(segmentsAnnouncedByMe.contains(segment)); - Assert.assertFalse("segment files shouldn't be deleted", segmentLoader.getSegmentsInTrash().contains(segment)); - - zkCoordinator.stop(); - } - - @Test - public void testLoadCache() throws Exception - { - List segments = Lists.newLinkedList(); - for (int i = 0; i < COUNT; ++i) { - segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-01"))); - segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-02"))); - segments.add(makeSegment("test" + i, "2", Intervals.of("P1d/2011-04-02"))); - segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-03"))); - segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-04"))); - segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-05"))); - segments.add(makeSegment("test" + i, "2", Intervals.of("PT1h/2011-04-04T01"))); - segments.add(makeSegment("test" + i, "2", Intervals.of("PT1h/2011-04-04T02"))); - segments.add(makeSegment("test" + i, "2", Intervals.of("PT1h/2011-04-04T03"))); - segments.add(makeSegment("test" + i, "2", Intervals.of("PT1h/2011-04-04T05"))); - segments.add(makeSegment("test" + i, "2", Intervals.of("PT1h/2011-04-04T06"))); - segments.add(makeSegment("test_two" + i, "1", Intervals.of("P1d/2011-04-01"))); - segments.add(makeSegment("test_two" + i, "1", Intervals.of("P1d/2011-04-02"))); - } - Collections.sort(segments); - - for (DataSegment segment : segments) { - writeSegmentToCache(segment); - } - - checkCache(segments); - Assert.assertTrue(segmentManager.getDataSourceCounts().isEmpty()); - zkCoordinator.start(); - Assert.assertTrue(!segmentManager.getDataSourceCounts().isEmpty()); - for (int i = 0; i < COUNT; ++i) { - Assert.assertEquals(11L, segmentManager.getDataSourceCounts().get("test" + i).longValue()); - Assert.assertEquals(2L, segmentManager.getDataSourceCounts().get("test_two" + i).longValue()); - } - Assert.assertEquals(13 * COUNT, announceCount.get()); - zkCoordinator.stop(); - - for (DataSegment segment : segments) { - deleteSegmentFromCache(segment); - } - - Assert.assertEquals(0, infoDir.listFiles().length); - Assert.assertTrue(infoDir.delete()); - } - - private DataSegment makeSegment(String dataSource, String version, Interval interval) - { - return new DataSegment( - dataSource, - interval, - version, - ImmutableMap.of("version", version, "interval", interval, "cacheDir", infoDir), + EmittingLogger.registerEmitter(new NoopServiceEmitter()); + DataSegment segment = new DataSegment( + "test", + Intervals.of("P1d/2011-04-02"), + "v0", + ImmutableMap.of("version", "v0", "interval", Intervals.of("P1d/2011-04-02"), "cacheDir", "/no"), Arrays.asList("dim1", "dim2", "dim3"), Arrays.asList("metric1", "metric2"), NoneShardSpec.instance(), IndexIO.CURRENT_VERSION_ID, 123L ); - } - - private void writeSegmentToCache(final DataSegment segment) throws IOException - { - if (!infoDir.exists()) { - infoDir.mkdir(); - } - - File segmentInfoCacheFile = new File( - infoDir, - segment.getIdentifier() - ); - try { - jsonMapper.writeValue(segmentInfoCacheFile, segment); - } - catch (IOException e) { - throw new RuntimeException(e); - } - - Assert.assertTrue(segmentInfoCacheFile.exists()); - } - - private void deleteSegmentFromCache(final DataSegment segment) throws IOException - { - File segmentInfoCacheFile = new File( - infoDir, - segment.getIdentifier() - ); - if (segmentInfoCacheFile.exists()) { - segmentInfoCacheFile.delete(); - } - Assert.assertTrue(!segmentInfoCacheFile.exists()); - } + CountDownLatch loadLatch = new CountDownLatch(1); + CountDownLatch dropLatch = new CountDownLatch(1); - private void checkCache(List segments) throws IOException - { - Assert.assertTrue(infoDir.exists()); - File[] files = infoDir.listFiles(); + SegmentLoadDropHandler segmentLoadDropHandler = EasyMock.createStrictMock(SegmentLoadDropHandler.class); - List sortedFiles = Lists.newArrayList(files); - Collections.sort(sortedFiles); - - Assert.assertEquals(segments.size(), sortedFiles.size()); - for (int i = 0; i < sortedFiles.size(); i++) { - DataSegment segment = jsonMapper.readValue(sortedFiles.get(i), DataSegment.class); - Assert.assertEquals(segments.get(i), segment); - } - } - - @Test - public void testInjector() throws Exception - { - Injector injector = Guice.createInjector( - new Module() + EasyMock.expect( + segmentLoadDropHandler.getSyncLoadingHanlder() + ).andReturn( + new DataSegmentChangeHandler() { + @Override - public void configure(Binder binder) + public void addSegment(DataSegment s, DataSegmentChangeCallback callback) { - binder.bind(ObjectMapper.class).toInstance(jsonMapper); - binder.bind(SegmentLoaderConfig.class).toInstance( - new SegmentLoaderConfig() - { - @Override - public File getInfoDir() - { - return infoDir; - } - - @Override - public int getNumLoadingThreads() - { - return 5; - } - - @Override - public int getAnnounceIntervalMillis() - { - return 50; - } - } - ); - binder.bind(ZkPathsConfig.class).toInstance( - new ZkPathsConfig() - { - @Override - public String getBase() - { - return "/druid"; - } - } - ); - binder.bind(DruidServerMetadata.class) - .toInstance(new DruidServerMetadata("dummyServer", "dummyHost", null, 0, ServerType.HISTORICAL, "normal", 0)); - binder.bind(DataSegmentAnnouncer.class).toInstance(announcer); - binder.bind(DataSegmentServerAnnouncer.class).toInstance(EasyMock.createNiceMock(DataSegmentServerAnnouncer.class)); - binder.bind(CuratorFramework.class).toInstance(curator); - binder.bind(ServerManager.class).toInstance(serverManager); - binder.bind(SegmentManager.class).toInstance(segmentManager); - binder.bind(ScheduledExecutorFactory.class).toInstance(ScheduledExecutors.createFactory(new Lifecycle())); + if (segment.getIdentifier().equals(s.getIdentifier())) { + loadLatch.countDown(); + callback.execute(); + } } + @Override + public void removeSegment(DataSegment s, DataSegmentChangeCallback callback) + { + if (segment.getIdentifier().equals(s.getIdentifier())) { + dropLatch.countDown(); + callback.execute(); + } + } } ); + EasyMock.replay(segmentLoadDropHandler); - ZkCoordinator zkCoordinator = injector.getInstance(ZkCoordinator.class); + zkCoordinator = new ZkCoordinator( + segmentLoadDropHandler, + jsonMapper, + zkPaths, + me, + curator + ); + zkCoordinator.start(); - List segments = Lists.newLinkedList(); - for (int i = 0; i < COUNT; ++i) { - segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-01"))); - segments.add(makeSegment("test" + i, "1", Intervals.of("P1d/2011-04-02"))); - segments.add(makeSegment("test" + i, "2", Intervals.of("P1d/2011-04-02"))); - segments.add(makeSegment("test_two" + i, "1", Intervals.of("P1d/2011-04-01"))); - segments.add(makeSegment("test_two" + i, "1", Intervals.of("P1d/2011-04-02"))); - } - Collections.sort(segments); + String segmentZkPath = ZKPaths.makePath(zkPaths.getLoadQueuePath(), me.getName(), segment.getIdentifier()); - for (DataSegment segment : segments) { - writeSegmentToCache(segment); - } + curator.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath( + segmentZkPath, jsonMapper.writeValueAsBytes(new SegmentChangeRequestLoad(segment))); - checkCache(segments); - Assert.assertTrue(segmentManager.getDataSourceCounts().isEmpty()); + loadLatch.await(); - zkCoordinator.start(); - Assert.assertTrue(!segmentManager.getDataSourceCounts().isEmpty()); - for (int i = 0; i < COUNT; ++i) { - Assert.assertEquals(3L, segmentManager.getDataSourceCounts().get("test" + i).longValue()); - Assert.assertEquals(2L, segmentManager.getDataSourceCounts().get("test_two" + i).longValue()); + while (curator.checkExists().forPath(segmentZkPath) != null) { + Thread.sleep(100); } - Assert.assertEquals(5 * COUNT, announceCount.get()); - zkCoordinator.stop(); - for (DataSegment segment : segments) { - deleteSegmentFromCache(segment); + curator.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath( + segmentZkPath, jsonMapper.writeValueAsBytes(new SegmentChangeRequestDrop(segment))); + + dropLatch.await(); + + while (curator.checkExists().forPath(segmentZkPath) != null) { + Thread.sleep(100); } - Assert.assertEquals(0, infoDir.listFiles().length); - Assert.assertTrue(infoDir.delete()); + zkCoordinator.stop(); + + EasyMock.verify(segmentLoadDropHandler); } } diff --git a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorTest.java b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorTest.java index 5157dba8040a..c2009ddfeb54 100644 --- a/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorTest.java +++ b/server/src/test/java/io/druid/server/coordinator/DruidCoordinatorTest.java @@ -141,7 +141,7 @@ public void setUp() throws Exception true, Execs.singleThreaded("coordinator_test_path_children_cache-%d") ); - loadQueuePeon = new LoadQueuePeon( + loadQueuePeon = new CuratorLoadQueuePeon( curator, LOADPATH, objectMapper, diff --git a/server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java b/server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java new file mode 100644 index 000000000000..60beec89f41f --- /dev/null +++ b/server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java @@ -0,0 +1,210 @@ +/* + * Licensed to Metamarkets Group Inc. (Metamarkets) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Metamarkets licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package io.druid.server.coordinator; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.metamx.http.client.HttpClient; +import com.metamx.http.client.Request; +import com.metamx.http.client.response.HttpResponseHandler; +import io.druid.TestUtil; +import io.druid.concurrent.Execs; +import io.druid.discovery.DiscoveryDruidNode; +import io.druid.discovery.DruidNodeDiscovery; +import io.druid.java.util.common.Intervals; +import io.druid.java.util.common.Pair; +import io.druid.java.util.common.RE; +import io.druid.server.coordination.DataSegmentChangeRequest; +import io.druid.server.coordination.SegmentLoadDropHandler; +import io.druid.timeline.DataSegment; +import org.jboss.netty.buffer.ChannelBuffers; +import org.jboss.netty.handler.codec.http.DefaultHttpResponse; +import org.jboss.netty.handler.codec.http.HttpResponse; +import org.jboss.netty.handler.codec.http.HttpResponseStatus; +import org.jboss.netty.handler.codec.http.HttpVersion; +import org.joda.time.Duration; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; + +/** + */ +public class HttpLoadQueuePeonTest +{ + @Test(timeout = 10000) + public void testSimple() throws Exception + { + final DataSegment segment1 = new DataSegment( + "test1", Intervals.of("2014/2015"), "v1", + null, null, null, null, 0, 0 + ); + + final DataSegment segment2 = new DataSegment( + "test2", Intervals.of("2014/2015"), "v1", + null, null, null, null, 0, 0 + ); + + final DataSegment segment3 = new DataSegment( + "test3", Intervals.of("2014/2015"), "v1", + null, null, null, null, 0, 0 + ); + + final DataSegment segment4 = new DataSegment( + "test4", Intervals.of("2014/2015"), "v1", + null, null, null, null, 0, 0 + ); + + HttpLoadQueuePeon httpLoadQueuePeon = new HttpLoadQueuePeon( + "http://dummy:4000", + TestUtil.MAPPER, + new TestHttpClient(), + new TestDruidCoordinatorConfig(null, null, null, null, null, null, 10, null, false, false, Duration.ZERO) { + @Override + public int getHttpLoadQueuePeonBatchSize() + { + return 2; + } + }, + Executors.newScheduledThreadPool( + 2, + Execs.makeThreadFactory("HttpLoadQueuePeonTest-%s") + ), + Execs.singleThreaded("HttpLoadQueuePeonTest") + ); + + httpLoadQueuePeon.start(); + + Map latches = ImmutableMap.of( + segment1.getIdentifier(), new CountDownLatch(1), + segment2.getIdentifier(), new CountDownLatch(1), + segment3.getIdentifier(), new CountDownLatch(1), + segment4.getIdentifier(), new CountDownLatch(1) + ); + + httpLoadQueuePeon.dropSegment(segment1, new LoadPeonCallback() + { + @Override + public void execute() + { + latches.get(segment1.getIdentifier()).countDown(); + } + }); + + httpLoadQueuePeon.loadSegment(segment2, new LoadPeonCallback() + { + @Override + public void execute() + { + latches.get(segment2.getIdentifier()).countDown(); + } + }); + + httpLoadQueuePeon.dropSegment(segment3, new LoadPeonCallback() + { + @Override + public void execute() + { + latches.get(segment3.getIdentifier()).countDown(); + } + }); + + httpLoadQueuePeon.loadSegment(segment4, new LoadPeonCallback() + { + @Override + public void execute() + { + latches.get(segment4.getIdentifier()).countDown(); + } + }); + + latches.get(segment1.getIdentifier()).await(); + latches.get(segment2.getIdentifier()).await(); + latches.get(segment3.getIdentifier()).await(); + latches.get(segment4.getIdentifier()).await(); + + httpLoadQueuePeon.stop(); + } + + private static class TestDruidNodeDiscovery implements DruidNodeDiscovery + { + Listener listener; + + @Override + public Collection getAllNodes() + { + throw new UnsupportedOperationException("Not Implemented."); + } + + @Override + public void registerListener(Listener listener) + { + listener.nodesAdded(ImmutableList.of()); + this.listener = listener; + } + } + + private static class TestHttpClient implements HttpClient + { + AtomicInteger requestNum = new AtomicInteger(0); + + @Override + public ListenableFuture go( + Request request, HttpResponseHandler httpResponseHandler + ) + { + throw new UnsupportedOperationException("Not Implemented."); + } + + @Override + public ListenableFuture go( + Request request, HttpResponseHandler httpResponseHandler, Duration duration + ) + { + HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); + httpResponse.setContent(ChannelBuffers.buffer(0)); + httpResponseHandler.handleResponse(httpResponse); + try { + List changeRequests = TestUtil.MAPPER.readValue( + request.getContent().array(), new TypeReference>() {} + ); + + List> statuses = new ArrayList<>(changeRequests.size()); + for (DataSegmentChangeRequest cr : changeRequests) { + statuses.add(Pair.of(cr, SegmentLoadDropHandler.Status.SUCCESS)); + } + return (ListenableFuture) Futures.immediateFuture(new ByteArrayInputStream(TestUtil.MAPPER.writerWithType( + HttpLoadQueuePeon.RESPONSE_ENTITY_TYPE_REF).writeValueAsBytes(statuses))); + } + catch (Exception ex) { + throw new RE(ex, "Unexpected exception."); + } + } + } +} diff --git a/server/src/test/java/io/druid/server/coordinator/LoadQueuePeonTest.java b/server/src/test/java/io/druid/server/coordinator/LoadQueuePeonTest.java index 108e1ac5c974..e5ce7d391783 100644 --- a/server/src/test/java/io/druid/server/coordinator/LoadQueuePeonTest.java +++ b/server/src/test/java/io/druid/server/coordinator/LoadQueuePeonTest.java @@ -82,7 +82,7 @@ public void testMultipleLoadDropSegments() throws Exception final AtomicInteger requestSignalIdx = new AtomicInteger(0); final AtomicInteger segmentSignalIdx = new AtomicInteger(0); - loadQueuePeon = new LoadQueuePeon( + loadQueuePeon = new CuratorLoadQueuePeon( curator, LOAD_QUEUE_PATH, jsonMapper, @@ -289,7 +289,7 @@ public void testFailAssign() throws Exception final CountDownLatch loadRequestSignal = new CountDownLatch(1); final CountDownLatch segmentLoadedSignal = new CountDownLatch(1); - loadQueuePeon = new LoadQueuePeon( + loadQueuePeon = new CuratorLoadQueuePeon( curator, LOAD_QUEUE_PATH, jsonMapper, diff --git a/server/src/test/java/io/druid/server/coordinator/LoadQueuePeonTester.java b/server/src/test/java/io/druid/server/coordinator/LoadQueuePeonTester.java index f394ad04b2ea..7b3f2808cab9 100644 --- a/server/src/test/java/io/druid/server/coordinator/LoadQueuePeonTester.java +++ b/server/src/test/java/io/druid/server/coordinator/LoadQueuePeonTester.java @@ -23,7 +23,7 @@ import java.util.concurrent.ConcurrentSkipListSet; -public class LoadQueuePeonTester extends LoadQueuePeon +public class LoadQueuePeonTester extends CuratorLoadQueuePeon { private final ConcurrentSkipListSet segmentsToLoad = new ConcurrentSkipListSet(); diff --git a/server/src/test/java/io/druid/server/metrics/HistoricalMetricsMonitorTest.java b/server/src/test/java/io/druid/server/metrics/HistoricalMetricsMonitorTest.java index f97440bdad56..652600a7d9ba 100644 --- a/server/src/test/java/io/druid/server/metrics/HistoricalMetricsMonitorTest.java +++ b/server/src/test/java/io/druid/server/metrics/HistoricalMetricsMonitorTest.java @@ -29,7 +29,7 @@ import io.druid.client.DruidServerConfig; import io.druid.java.util.common.Intervals; import io.druid.server.SegmentManager; -import io.druid.server.coordination.ZkCoordinator; +import io.druid.server.coordination.SegmentLoadDropHandler; import io.druid.timeline.DataSegment; import org.easymock.Capture; import org.easymock.CaptureType; @@ -48,7 +48,7 @@ public class HistoricalMetricsMonitorTest extends EasyMockSupport { private DruidServerConfig druidServerConfig; private SegmentManager segmentManager; - private ZkCoordinator zkCoordinator; + private SegmentLoadDropHandler segmentLoadDropMgr; private ServiceEmitter serviceEmitter; @Before @@ -56,7 +56,7 @@ public void setUp() { druidServerConfig = EasyMock.createStrictMock(DruidServerConfig.class); segmentManager = EasyMock.createStrictMock(SegmentManager.class); - zkCoordinator = EasyMock.createStrictMock(ZkCoordinator.class); + segmentLoadDropMgr = EasyMock.createStrictMock(SegmentLoadDropHandler.class); serviceEmitter = EasyMock.createStrictMock(ServiceEmitter.class); } @@ -81,7 +81,7 @@ public void testSimple() final String tier = "tier"; EasyMock.expect(druidServerConfig.getMaxSize()).andReturn(maxSize).once(); - EasyMock.expect(zkCoordinator.getPendingDeleteSnapshot()).andReturn(ImmutableList.of(dataSegment)).once(); + EasyMock.expect(segmentLoadDropMgr.getPendingDeleteSnapshot()).andReturn(ImmutableList.of(dataSegment)).once(); EasyMock.expect(druidServerConfig.getTier()).andReturn(tier).once(); EasyMock.expect(druidServerConfig.getPriority()).andReturn(priority).once(); EasyMock.expect(segmentManager.getDataSourceSizes()).andReturn(ImmutableMap.of(dataSource, size)); @@ -95,16 +95,16 @@ public void testSimple() final HistoricalMetricsMonitor monitor = new HistoricalMetricsMonitor( druidServerConfig, segmentManager, - zkCoordinator + segmentLoadDropMgr ); final Capture> eventCapture = EasyMock.newCapture(CaptureType.ALL); serviceEmitter.emit(EasyMock.capture(eventCapture)); EasyMock.expectLastCall().times(5); - EasyMock.replay(druidServerConfig, segmentManager, zkCoordinator, serviceEmitter); + EasyMock.replay(druidServerConfig, segmentManager, segmentLoadDropMgr, serviceEmitter); monitor.doMonitor(serviceEmitter); - EasyMock.verify(druidServerConfig, segmentManager, zkCoordinator, serviceEmitter); + EasyMock.verify(druidServerConfig, segmentManager, segmentLoadDropMgr, serviceEmitter); final String host = "host"; final String service = "service"; diff --git a/services/src/main/java/io/druid/cli/CliCoordinator.java b/services/src/main/java/io/druid/cli/CliCoordinator.java index 3aff7ddf15e7..f2107d54c621 100644 --- a/services/src/main/java/io/druid/cli/CliCoordinator.java +++ b/services/src/main/java/io/druid/cli/CliCoordinator.java @@ -28,6 +28,7 @@ import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.name.Names; +import com.metamx.http.client.HttpClient; import io.airlift.airline.Command; import io.druid.audit.AuditManager; import io.druid.client.CoordinatorServerView; @@ -42,6 +43,7 @@ import io.druid.guice.LifecycleModule; import io.druid.guice.ManageLifecycle; import io.druid.guice.annotations.CoordinatorIndexingServiceHelper; +import io.druid.guice.annotations.Global; import io.druid.java.util.common.concurrent.ScheduledExecutorFactory; import io.druid.java.util.common.logger.Logger; import io.druid.metadata.MetadataRuleManager; @@ -74,6 +76,7 @@ import io.druid.server.http.RulesResource; import io.druid.server.http.ServersResource; import io.druid.server.http.TiersResource; +import io.druid.server.initialization.ZkPathsConfig; import io.druid.server.initialization.jetty.JettyServerInitializer; import io.druid.server.lookup.cache.LookupCoordinatorManager; import io.druid.server.lookup.cache.LookupCoordinatorManagerConfig; @@ -221,12 +224,14 @@ public LoadQueueTaskMaster getLoadQueueTaskMaster( CuratorFramework curator, ObjectMapper jsonMapper, ScheduledExecutorFactory factory, - DruidCoordinatorConfig config + DruidCoordinatorConfig config, + @Global HttpClient httpClient, + ZkPathsConfig zkPaths ) { return new LoadQueueTaskMaster( curator, jsonMapper, factory.create(1, "Master-PeonExec--%d"), - Executors.newSingleThreadExecutor(), config + Executors.newSingleThreadExecutor(), config, httpClient, zkPaths ); } } diff --git a/services/src/main/java/io/druid/cli/CliPeon.java b/services/src/main/java/io/druid/cli/CliPeon.java index 0ebef587ca37..a205dfab01b3 100644 --- a/services/src/main/java/io/druid/cli/CliPeon.java +++ b/services/src/main/java/io/druid/cli/CliPeon.java @@ -51,6 +51,7 @@ import io.druid.guice.QueryableModule; import io.druid.guice.QueryablePeonModule; import io.druid.guice.annotations.Json; +import io.druid.guice.annotations.Smile; import io.druid.indexing.common.RetryPolicyConfig; import io.druid.indexing.common.RetryPolicyFactory; import io.druid.indexing.common.TaskToolboxFactory; @@ -87,6 +88,7 @@ import io.druid.segment.realtime.plumber.CoordinatorBasedSegmentHandoffNotifierConfig; import io.druid.segment.realtime.plumber.CoordinatorBasedSegmentHandoffNotifierFactory; import io.druid.segment.realtime.plumber.SegmentHandoffNotifierFactory; +import io.druid.server.coordination.BatchDataSegmentAnnouncer; import io.druid.server.coordination.ServerType; import io.druid.server.http.SegmentListerResource; import io.druid.server.initialization.jetty.ChatHandlerServerModule; @@ -94,6 +96,7 @@ import io.druid.server.metrics.DataSourceTaskIdHolder; import org.eclipse.jetty.server.Server; +import javax.annotation.Nullable; import java.io.File; import java.io.IOException; import java.util.Arrays; @@ -266,6 +269,21 @@ public String getTaskIDFromTask(final Task task) { return task.getId(); } + + @Provides + public SegmentListerResource getSegmentListerResource( + @Json ObjectMapper jsonMapper, + @Smile ObjectMapper smileMapper, + @Nullable BatchDataSegmentAnnouncer announcer + ) + { + return new SegmentListerResource( + jsonMapper, + smileMapper, + announcer, + null + ); + } }, new QueryablePeonModule(), new IndexingServiceFirehoseModule(), From c8d1c0a4db13ff2e67fa372e5929459bca09857c Mon Sep 17 00:00:00 2001 From: Himanshu Gupta Date: Thu, 5 Oct 2017 15:17:01 -0500 Subject: [PATCH 9/9] address review comments --- .../java/io/druid/java/util/common/Pair.java | 7 +- .../coordination/SegmentLoadDropHandler.java | 180 +++++--- .../server/coordination/ZkCoordinator.java | 2 +- .../server/coordinator/HttpLoadQueuePeon.java | 411 ++++++++---------- .../server/http/SegmentListerResource.java | 10 +- .../jetty/JettyServerModule.java | 2 + .../SegmentLoadDropHandlerTest.java | 22 +- .../coordination/ZkCoordinatorTest.java | 59 +-- .../coordinator/HttpLoadQueuePeonTest.java | 5 +- 9 files changed, 355 insertions(+), 343 deletions(-) diff --git a/java-util/src/main/java/io/druid/java/util/common/Pair.java b/java-util/src/main/java/io/druid/java/util/common/Pair.java index f821eb599342..62142622c70c 100644 --- a/java-util/src/main/java/io/druid/java/util/common/Pair.java +++ b/java-util/src/main/java/io/druid/java/util/common/Pair.java @@ -19,7 +19,6 @@ package io.druid.java.util.common; -import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import java.util.Comparator; @@ -34,15 +33,13 @@ public static Pair of(T1 lhs, T2 rhs) return new Pair<>(lhs, rhs); } - @JsonProperty public final T1 lhs; - @JsonProperty public final T2 rhs; public Pair( - @JsonProperty("lhs") T1 lhs, - @JsonProperty("rhs") T2 rhs + T1 lhs, + T2 rhs ) { this.lhs = lhs; diff --git a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java index 8aada51f65d6..01bf0cdb8ed5 100644 --- a/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentLoadDropHandler.java @@ -19,6 +19,7 @@ package io.druid.server.coordination; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; @@ -28,6 +29,7 @@ import com.google.common.cache.CacheBuilder; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Queues; import com.google.common.util.concurrent.AbstractFuture; import com.google.common.util.concurrent.ListenableFuture; @@ -36,9 +38,7 @@ import com.metamx.emitter.EmittingLogger; import io.druid.concurrent.Execs; import io.druid.guice.ManageLifecycle; -import io.druid.java.util.common.IAE; import io.druid.java.util.common.ISE; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.lifecycle.LifecycleStart; import io.druid.java.util.common.lifecycle.LifecycleStop; import io.druid.segment.loading.SegmentLoaderConfig; @@ -50,7 +50,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -70,7 +69,7 @@ /** */ @ManageLifecycle -public class SegmentLoadDropHandler +public class SegmentLoadDropHandler implements DataSegmentChangeHandler { private static final EmittingLogger log = new EmittingLogger(SegmentLoadDropHandler.class); @@ -201,7 +200,7 @@ private void loadLocalCache() int ignored = 0; for (int i = 0; i < segmentsToLoad.length; i++) { File file = segmentsToLoad[i]; - log.info("Loading segment cache file [%d/%d][%s].", i, segmentsToLoad.length, file); + log.info("Loading segment cache file [%d/%d][%s].", i + 1, segmentsToLoad.length, file); try { final DataSegment segment = jsonMapper.readValue(file, DataSegment.class); @@ -278,7 +277,8 @@ private void loadSegment(DataSegment segment, DataSegmentChangeCallback callback } } - void addSegment(DataSegment segment, DataSegmentChangeCallback callback) + @Override + public void addSegment(DataSegment segment, DataSegmentChangeCallback callback) { Status result = null; try { @@ -300,7 +300,7 @@ each time when addSegment() is called, it has to wait for the lock in order to m segmentsToDelete.remove(segment); } } - loadSegment(segment, callback); + loadSegment(segment, DataSegmentChangeCallback.NOOP); try { announcer.announceSegment(segment); } @@ -317,7 +317,7 @@ each time when addSegment() is called, it has to wait for the lock in order to m result = Status.failed(e.getMessage()); } finally { - updateStatusAndResolveWaitingFutures(new SegmentChangeRequestLoad(segment), result); + updateRequestStatus(new SegmentChangeRequestLoad(segment), result); callback.execute(); } } @@ -346,7 +346,7 @@ public void run() try { log.info( "Loading segment[%d/%d][%s]", - counter.getAndIncrement(), + counter.incrementAndGet(), numSegments, segment.getIdentifier() ); @@ -399,12 +399,19 @@ public void run() } } - void removeSegment( + @Override + public void removeSegment(DataSegment segment, DataSegmentChangeCallback callback) + { + removeSegment(segment, callback, true); + } + + private void removeSegment( final DataSegment segment, final DataSegmentChangeCallback callback, final boolean scheduleDrop ) { + Status result = null; try { announcer.unannounceSegment(segment); segmentsToDelete.add(segment); @@ -414,7 +421,6 @@ void removeSegment( @Override public void run() { - Status result = null; try { synchronized (lock) { if (segmentsToDelete.remove(segment)) { @@ -426,17 +432,11 @@ public void run() } } } - - result = Status.SUCCESS; } catch (Exception e) { log.makeAlert(e, "Failed to remove segment! Possible resource leak!") .addData("segment", segment) .emit(); - result = Status.failed(e.getMessage()); - } - finally { - updateStatusAndResolveWaitingFutures(new SegmentChangeRequestDrop(segment), result); } } }; @@ -452,16 +452,20 @@ public void run() config.getDropSegmentDelayMillis(), TimeUnit.MILLISECONDS ); + } else { + runnable.run(); } + + result = Status.SUCCESS; } catch (Exception e) { log.makeAlert(e, "Failed to remove segment") .addData("segment", segment) .emit(); - - updateStatusAndResolveWaitingFutures(new SegmentChangeRequestDrop(segment), Status.failed(e.getMessage())); + result = Status.failed(e.getMessage()); } finally { + updateRequestStatus(new SegmentChangeRequestDrop(segment), result); callback.execute(); } } @@ -471,11 +475,11 @@ public Collection getPendingDeleteSnapshot() return ImmutableList.copyOf(segmentsToDelete); } - public ListenableFuture>> processBatch(List changeRequests) + public ListenableFuture> processBatch(List changeRequests) { boolean isAnyRequestDone = false; - Map> statuses = new HashMap<>(changeRequests.size()); + Map> statuses = Maps.newHashMapWithExpectedSize(changeRequests.size()); for (DataSegmentChangeRequest cr : changeRequests) { AtomicReference status = processRequest(cr); @@ -502,41 +506,54 @@ private AtomicReference processRequest(DataSegmentChangeRequest changeRe { synchronized (requestStatusesLock) { if (requestStatuses.getIfPresent(changeRequest) == null) { - if (changeRequest instanceof SegmentChangeRequestLoad) { - requestStatuses.put(changeRequest, new AtomicReference<>(Status.PENDING)); - exec.submit( - () -> SegmentLoadDropHandler.this.addSegment( - ((SegmentChangeRequestLoad) changeRequest).getSegment(), - DataSegmentChangeCallback.NOOP - ) - ); - } else if (changeRequest instanceof SegmentChangeRequestDrop) { - requestStatuses.put(changeRequest, new AtomicReference<>(Status.PENDING)); - SegmentLoadDropHandler.this.removeSegment( - ((SegmentChangeRequestDrop) changeRequest).getSegment(), - DataSegmentChangeCallback.NOOP, - true - ); - } else { - throw new IAE("Not a segment load/drop request[%s].", changeRequest.getClass().getName()); - } + changeRequest.go( + new DataSegmentChangeHandler() + { + @Override + public void addSegment(DataSegment segment, DataSegmentChangeCallback callback) + { + requestStatuses.put(changeRequest, new AtomicReference<>(Status.PENDING)); + exec.submit( + () -> SegmentLoadDropHandler.this.addSegment( + ((SegmentChangeRequestLoad) changeRequest).getSegment(), + () -> resolveWaitingFutures() + ) + ); + } + + @Override + public void removeSegment(DataSegment segment, DataSegmentChangeCallback callback) + { + requestStatuses.put(changeRequest, new AtomicReference<>(Status.PENDING)); + SegmentLoadDropHandler.this.removeSegment( + ((SegmentChangeRequestDrop) changeRequest).getSegment(), + () -> resolveWaitingFutures(), + true + ); + } + }, + () -> resolveWaitingFutures() + ); } return requestStatuses.getIfPresent(changeRequest); } } - private void updateStatusAndResolveWaitingFutures(DataSegmentChangeRequest chanteRequest, Status result) + private void updateRequestStatus(DataSegmentChangeRequest changeRequest, Status result) { if (result == null) { result = Status.failed("Unknown reason. Check server logs."); } synchronized (requestStatusesLock) { - AtomicReference statusRef = requestStatuses.getIfPresent(chanteRequest); + AtomicReference statusRef = requestStatuses.getIfPresent(changeRequest); if (statusRef != null) { statusRef.set(result); } } + } + private void resolveWaitingFutures() + { LinkedHashSet waitingFuturesCopy = new LinkedHashSet<>(); synchronized (waitingFutures) { waitingFuturesCopy.addAll(waitingFutures); @@ -547,31 +564,6 @@ private void updateStatusAndResolveWaitingFutures(DataSegmentChangeRequest chant } } - public DataSegmentChangeHandler getSyncLoadingHanlder() - { - return new DataSegmentChangeHandler() - { - @Override - public void addSegment(DataSegment segment, DataSegmentChangeCallback callback) - { - SegmentLoadDropHandler.this.addSegment(segment, callback); - } - - @Override - public void removeSegment(DataSegment segment, DataSegmentChangeCallback callback) - { - try { - SegmentLoadDropHandler.this.removeSegment(segment, callback, true); - } - catch (Exception ex) { - log.makeAlert(ex, "Failed to remove segment") - .addData("segment", segment) - .emit(); - } - } - }; - } - private static class BackgroundSegmentAnnouncer implements AutoCloseable { private static final EmittingLogger log = new EmittingLogger(BackgroundSegmentAnnouncer.class); @@ -704,7 +696,7 @@ public void close() } // Future with cancel() implementation to remove it from "waitingFutures" list - private static class CustomSettableFuture extends AbstractFuture>> + private static class CustomSettableFuture extends AbstractFuture> { private final LinkedHashSet waitingFutures; private final Map> statusRefs; @@ -720,12 +712,18 @@ private CustomSettableFuture( public void resolve() { - List> result = new ArrayList<>(statusRefs.size()); - statusRefs.forEach( - (request, statusRef) -> result.add(new Pair(request, statusRef.get())) - ); + synchronized (statusRefs) { + if (isDone()) { + return; + } + + List result = new ArrayList<>(statusRefs.size()); + statusRefs.forEach( + (request, statusRef) -> result.add(new DataSegmentChangeRequestAndStatus(request, statusRef.get())) + ); - super.set(result); + super.set(result); + } } @Override @@ -757,6 +755,7 @@ public enum STATE public final static Status SUCCESS = new Status(STATE.SUCCESS, null); public final static Status PENDING = new Status(STATE.PENDING, null); + @JsonCreator Status( @JsonProperty("state") STATE state, @JsonProperty("failureCause") String failureCause @@ -793,5 +792,42 @@ public String toString() '}'; } } + + public static class DataSegmentChangeRequestAndStatus + { + private final DataSegmentChangeRequest request; + private final Status status; + + @JsonCreator + public DataSegmentChangeRequestAndStatus( + @JsonProperty("request") DataSegmentChangeRequest request, + @JsonProperty("status") Status status + ) + { + this.request = request; + this.status = status; + } + + @JsonProperty + public DataSegmentChangeRequest getRequest() + { + return request; + } + + @JsonProperty + public Status getStatus() + { + return status; + } + + @Override + public String toString() + { + return "DataSegmentChangeRequestAndStatus{" + + "request=" + request + + ", status=" + status + + '}'; + } + } } diff --git a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java index 65854945af15..d181a1477129 100644 --- a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java +++ b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java @@ -62,7 +62,7 @@ public ZkCoordinator( CuratorFramework curator ) { - this.dataSegmentChangeHandler = loadDropHandler.getSyncLoadingHanlder(); + this.dataSegmentChangeHandler = loadDropHandler; this.jsonMapper = jsonMapper; this.zkPaths = zkPaths; this.me = me; diff --git a/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java b/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java index dff43824dcc3..1c9f7fca7f67 100644 --- a/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java +++ b/server/src/main/java/io/druid/server/coordinator/HttpLoadQueuePeon.java @@ -21,7 +21,9 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.base.Throwables; +import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; @@ -35,7 +37,6 @@ import com.metamx.http.client.response.ClientResponse; import com.metamx.http.client.response.InputStreamResponseHandler; import io.druid.java.util.common.ISE; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.concurrent.ScheduledExecutors; import io.druid.server.coordination.DataSegmentChangeCallback; import io.druid.server.coordination.DataSegmentChangeHandler; @@ -55,6 +56,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -63,25 +65,24 @@ import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ConcurrentSkipListSet; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; /** */ public class HttpLoadQueuePeon extends LoadQueuePeon { - public static final TypeReference REQUEST_ENTITY_TYPE_REF = new TypeReference>() {}; - public static final TypeReference RESPONSE_ENTITY_TYPE_REF = new TypeReference>>() {}; + public static final TypeReference REQUEST_ENTITY_TYPE_REF = new TypeReference>() + { + }; - private static final EmittingLogger log = new EmittingLogger(HttpLoadQueuePeon.class); + public static final TypeReference RESPONSE_ENTITY_TYPE_REF = new TypeReference>() + { + }; - private static final int DROP = 0; - private static final int LOAD = 1; + private static final EmittingLogger log = new EmittingLogger(HttpLoadQueuePeon.class); private final AtomicLong queuedSize = new AtomicLong(0); private final AtomicInteger failedAssignCount = new AtomicInteger(0); @@ -112,6 +113,8 @@ public class HttpLoadQueuePeon extends LoadQueuePeon private final AtomicBoolean mainLoopInProgress = new AtomicBoolean(false); private final ExecutorService callBackExecutor; + private final ObjectWriter requestBodyWriter; + public HttpLoadQueuePeon( String baseUrl, ObjectMapper jsonMapper, @@ -122,6 +125,7 @@ public HttpLoadQueuePeon( ) { this.jsonMapper = jsonMapper; + this.requestBodyWriter = jsonMapper.writerWithType(REQUEST_ENTITY_TYPE_REF); this.httpClient = httpClient; this.config = config; this.processingExecutor = processingExecutor; @@ -154,48 +158,54 @@ private void doSegmentManagement() List newRequests = new ArrayList<>(batchSize); synchronized (lock) { - Iterator> iter = segmentsToDrop.entrySet().iterator(); - while (batchSize-- > 0 && iter.hasNext()) { - Map.Entry entry = iter.next(); - newRequests.add(entry.getValue().getChangeRequest()); - entry.getValue().scheduleCancellationOnTimeout(); - } + Iterator> iter = Iterators.concat( + segmentsToDrop.entrySet().iterator(), + segmentsToLoad.entrySet().iterator() + ); - iter = segmentsToLoad.entrySet().iterator(); - while (batchSize-- > 0 && iter.hasNext()) { + while (batchSize > 0 && iter.hasNext()) { + batchSize--; Map.Entry entry = iter.next(); - newRequests.add(entry.getValue().getChangeRequest()); - entry.getValue().scheduleCancellationOnTimeout(); + if (entry.getValue().hasTimedOut()) { + entry.getValue().requestFailed("timed out"); + iter.remove(); + } else { + newRequests.add(entry.getValue().getChangeRequest()); + } } } - if (newRequests.size() > 0) { - try { - log.debug("Sending [%d] load/drop requests to Server[%s].", newRequests.size(), serverId); - BytesAccumulatingResponseHandler responseHandler = new BytesAccumulatingResponseHandler(); - ListenableFuture future = httpClient.go( - new Request(HttpMethod.POST, changeRequestURL) - .addHeader(HttpHeaders.Names.ACCEPT, MediaType.APPLICATION_JSON) - .addHeader(HttpHeaders.Names.CONTENT_TYPE, MediaType.APPLICATION_JSON) - .setContent(jsonMapper.writerWithType(REQUEST_ENTITY_TYPE_REF).writeValueAsBytes(newRequests)), - responseHandler, - new Duration(config.getHttpLoadQueuePeonHostTimeout().getMillis() + 5000) - ); - - Futures.addCallback( - future, - new FutureCallback() - { - @Override - public void onSuccess(InputStream result) - { - boolean scheduleNextRunImmediately = true; + if (newRequests.size() == 0) { + mainLoopInProgress.set(false); + return; + } + try { + log.debug("Sending [%d] load/drop requests to Server[%s].", newRequests.size(), serverId); + BytesAccumulatingResponseHandler responseHandler = new BytesAccumulatingResponseHandler(); + ListenableFuture future = httpClient.go( + new Request(HttpMethod.POST, changeRequestURL) + .addHeader(HttpHeaders.Names.ACCEPT, MediaType.APPLICATION_JSON) + .addHeader(HttpHeaders.Names.CONTENT_TYPE, MediaType.APPLICATION_JSON) + .setContent(requestBodyWriter.writeValueAsBytes(newRequests)), + responseHandler, + new Duration(config.getHttpLoadQueuePeonHostTimeout().getMillis() + 5000) + ); + + Futures.addCallback( + future, + new FutureCallback() + { + @Override + public void onSuccess(InputStream result) + { + boolean scheduleNextRunImmediately = true; + try { if (responseHandler.status == HttpServletResponse.SC_NO_CONTENT) { log.debug("Received NO CONTENT reseponse from [%s]", serverId); } else if (HttpServletResponse.SC_OK == responseHandler.status) { try { - List> statuses = jsonMapper.readValue( + List statuses = jsonMapper.readValue( result, RESPONSE_ENTITY_TYPE_REF ); @@ -204,127 +214,105 @@ public void onSuccess(InputStream result) return; } - for (Pair e : statuses) { - switch (e.rhs.getState()) { + for (SegmentLoadDropHandler.DataSegmentChangeRequestAndStatus e : statuses) { + switch (e.getStatus().getState()) { case SUCCESS: case FAILED: - e.lhs.go( - new DataSegmentChangeHandler() - { - @Override - public void addSegment(DataSegment segment, DataSegmentChangeCallback callback) - { - SegmentHolder holder = segmentsToLoad.remove(segment); - if (holder == null) { - return; - } - - if (e.rhs.getState() - == SegmentLoadDropHandler.Status.STATE.FAILED) { - holder.requestCompleted( - true, - StringUtils.safeFormat( - "Server[%s] Failed segment[%s] LOAD request with cause [%s].", - serverId, - segment.getIdentifier(), - e.rhs.getFailureCause() - ) - ); - } else { - holder.requestCompleted(false, null); - } - - } - - @Override - public void removeSegment(DataSegment segment, DataSegmentChangeCallback callback) - { - SegmentHolder holder = segmentsToDrop.remove(segment); - if (holder == null) { - return; - } - - if (e.rhs.getState() - == SegmentLoadDropHandler.Status.STATE.FAILED) { - holder.requestCompleted( - true, - StringUtils.safeFormat( - "Server[%s] Failed segment[%s] DROP request with cause [%s].", - serverId, - segment.getIdentifier(), - e.rhs.getFailureCause() - ) - ); - } else { - holder.requestCompleted(false, null); - } - } - }, null - ); + handleResponseStatus(e.getRequest(), e.getStatus()); break; case PENDING: - log.debug("[%s]Segment request [%s] is pending.", serverId, e.lhs); + log.debug("[%s]Segment request [%s] is pending.", serverId, e.getRequest()); break; default: scheduleNextRunImmediately = false; - log.error("WTF! Server[%s] returned unknown state in status[%s].", serverId, e.rhs); + log.error("WTF! Server[%s] returned unknown state in status[%s].", serverId, e.getStatus()); } } } } catch (Exception ex) { scheduleNextRunImmediately = false; - onFailure(ex); - } - finally { - mainLoopInProgress.set(false); + logRequestFailure(ex); } } else { scheduleNextRunImmediately = false; - onFailure(new RE("Unexpected Response Status.")); + logRequestFailure(new RE("Unexpected Response Status.")); } + } + finally { + mainLoopInProgress.set(false); if (scheduleNextRunImmediately) { processingExecutor.execute(HttpLoadQueuePeon.this::doSegmentManagement); } } + } - @Override - public void onFailure(Throwable t) - { - try { - log.info( - "Request[%s] Failed with code[%s] and status[%s]. Reason[%s]. Error [%s]", - changeRequestURL, - responseHandler.status, - responseHandler.description, - t.getMessage() - ); - log.error( - t, - "Request[%s] Failed with code[%s] and status[%s]. Reason[%s].", - changeRequestURL, - responseHandler.status, - responseHandler.description - ); - } - finally { - mainLoopInProgress.set(false); - } + @Override + public void onFailure(Throwable t) + { + try { + logRequestFailure(t); } - }, - processingExecutor - ); - } - catch (Throwable th) { - mainLoopInProgress.set(false); - } - } else { - log.debug("[%s] No segments to load/drop."); + finally { + mainLoopInProgress.set(false); + } + } + + private void logRequestFailure(Throwable t) + { + log.error( + t, + "Request[%s] Failed with code[%s] and status[%s]. Reason[%s].", + changeRequestURL, + responseHandler.status, + responseHandler.description + ); + } + }, + processingExecutor + ); + } + catch (Throwable th) { + log.error(th, "Error sending load/drop request to [%s].", serverId); mainLoopInProgress.set(false); } } + private void handleResponseStatus(DataSegmentChangeRequest changeRequest, SegmentLoadDropHandler.Status status) + { + changeRequest.go( + new DataSegmentChangeHandler() + { + @Override + public void addSegment(DataSegment segment, DataSegmentChangeCallback callback) + { + updateSuccessOrFailureInHolder(segmentsToLoad.remove(segment), status); + } + + @Override + public void removeSegment(DataSegment segment, DataSegmentChangeCallback callback) + { + updateSuccessOrFailureInHolder(segmentsToDrop.remove(segment), status); + } + + private void updateSuccessOrFailureInHolder(SegmentHolder holder, SegmentLoadDropHandler.Status status) + { + if (holder == null) { + return; + } + + if (status.getState() + == SegmentLoadDropHandler.Status.STATE.FAILED) { + holder.requestFailed(status.getFailureCause()); + } else { + holder.requestSucceeded(); + } + } + }, null + ); + } + @Override public void start() { @@ -367,11 +355,11 @@ public void stop() stopped = true; for (SegmentHolder holder : segmentsToDrop.values()) { - holder.requestCompleted(false, null); + holder.requestSucceeded(); } for (SegmentHolder holder : segmentsToLoad.values()) { - holder.requestCompleted(false, null); + holder.requestSucceeded(); } segmentsToDrop.clear(); @@ -389,7 +377,7 @@ public void loadSegment(DataSegment segment, LoadPeonCallback callback) if (holder == null) { log.info("Server[%s] to load segment[%s] queued.", serverId, segment.getIdentifier()); - segmentsToLoad.put(segment, new SegmentHolder(segment, LOAD, callback)); + segmentsToLoad.put(segment, new LoadSegmentHolder(segment, callback)); processingExecutor.execute(this::doSegmentManagement); } else { holder.addCallback(callback); @@ -405,7 +393,7 @@ public void dropSegment(DataSegment segment, LoadPeonCallback callback) if (holder == null) { log.info("Server[%s] to drop segment[%s] queued.", serverId, segment.getIdentifier()); - segmentsToDrop.put(segment, new SegmentHolder(segment, DROP, callback)); + segmentsToDrop.put(segment, new DropSegmentHolder(segment, callback)); processingExecutor.execute(this::doSegmentManagement); } else { holder.addCallback(callback); @@ -416,13 +404,13 @@ public void dropSegment(DataSegment segment, LoadPeonCallback callback) @Override public Set getSegmentsToLoad() { - return segmentsToLoad.keySet(); + return Collections.unmodifiableSet(segmentsToLoad.keySet()); } @Override public Set getSegmentsToDrop() { - return segmentsToDrop.keySet(); + return Collections.unmodifiableSet(segmentsToDrop.keySet()); } @Override @@ -458,45 +446,30 @@ public int getNumberOfSegmentsInQueue() @Override public Set getSegmentsMarkedToDrop() { - return segmentsMarkedToDrop; + return Collections.unmodifiableSet(segmentsMarkedToDrop); } - private class SegmentHolder + private abstract class SegmentHolder { private final DataSegment segment; private final DataSegmentChangeRequest changeRequest; - private final int type; private final List callbacks = Lists.newArrayList(); + // Time when this request was sent to target server the first time. + private volatile long scheduleTime = -1; + private SegmentHolder( DataSegment segment, - int type, + DataSegmentChangeRequest changeRequest, LoadPeonCallback callback ) { this.segment = segment; - this.type = type; - this.changeRequest = (type == LOAD) - ? new SegmentChangeRequestLoad(segment) - : new SegmentChangeRequestDrop(segment); + this.changeRequest = changeRequest; if (callback != null) { this.callbacks.add(callback); } - - if (type == LOAD) { - queuedSize.addAndGet(segment.getSize()); - } - } - - public DataSegment getSegment() - { - return segment; - } - - public int getType() - { - return type; } public void addCallback(LoadPeonCallback newCallback) @@ -508,76 +481,52 @@ public void addCallback(LoadPeonCallback newCallback) } } + public DataSegment getSegment() + { + return segment; + } + public DataSegmentChangeRequest getChangeRequest() { return changeRequest; } - AtomicReference cancellationFutureRef = new AtomicReference<>(); - - public void scheduleCancellationOnTimeout() + public boolean hasTimedOut() { - cancellationFutureRef.updateAndGet( - (curr) -> { - if (curr == null) { - return processingExecutor.schedule( - () -> { - synchronized (lock) { - if (type == LOAD) { - segmentsToLoad.remove(segment); - } else { - segmentsToDrop.remove(segment); - } - requestCompleted( - true, - StringUtils.safeFormat( - "Server[%s] failed to complete request[%s] within timeout[%s millis].", - serverId, - config.getLoadTimeoutDelay().getMillis() - ) - ); - } - }, - config.getLoadTimeoutDelay().getMillis(), - TimeUnit.MILLISECONDS - ); - } else { - return curr; - } - } - ); + if (scheduleTime < 0) { + scheduleTime = System.currentTimeMillis(); + return false; + } else if (System.currentTimeMillis() - scheduleTime > config.getLoadTimeoutDelay().getMillis()) { + return true; + } else { + return false; + } } - public void requestCompleted(boolean failed, String failMsg) + public void requestSucceeded() { - synchronized (lock) { - Future cancellationFuture = cancellationFutureRef.get(); - - if (cancellationFuture != null && cancellationFuture.isDone()) { - return; + callBackExecutor.execute(() -> { + for (LoadPeonCallback callback : callbacks) { + if (callback != null) { + callback.execute(); + } } + }); + } - if (cancellationFuture != null) { - cancellationFuture.cancel(true); - } + public void requestFailed(String failureCause) + { + log.error( + "Server[%s] Failed segment[%s] request[%s] with cause [%s].", + serverId, + segment.getIdentifier(), + changeRequest.getClass().getSimpleName(), + failureCause + ); - if (failed) { - log.error(failMsg); - failedAssignCount.getAndIncrement(); - } + failedAssignCount.getAndIncrement(); - if (type == LOAD) { - queuedSize.addAndGet(-segment.getSize()); - } - - callBackExecutor.execute(() -> { - for (LoadPeonCallback callback : callbacks) { - if (callback != null) { - callback.execute(); - } - } - }); - } + requestSucceeded(); } @Override @@ -587,6 +536,30 @@ public String toString() } } + private class LoadSegmentHolder extends SegmentHolder + { + public LoadSegmentHolder(DataSegment segment, LoadPeonCallback callback) + { + super(segment, new SegmentChangeRequestLoad(segment), callback); + queuedSize.addAndGet(segment.getSize()); + } + + @Override + public void requestSucceeded() + { + queuedSize.addAndGet(-getSegment().getSize()); + super.requestSucceeded(); + } + } + + private class DropSegmentHolder extends SegmentHolder + { + public DropSegmentHolder(DataSegment segment, LoadPeonCallback callback) + { + super(segment, new SegmentChangeRequestDrop(segment), callback); + } + } + private static class BytesAccumulatingResponseHandler extends InputStreamResponseHandler { private int status; diff --git a/server/src/main/java/io/druid/server/http/SegmentListerResource.java b/server/src/main/java/io/druid/server/http/SegmentListerResource.java index 9f545dcc73ec..600806cc7e13 100644 --- a/server/src/main/java/io/druid/server/http/SegmentListerResource.java +++ b/server/src/main/java/io/druid/server/http/SegmentListerResource.java @@ -29,7 +29,6 @@ import com.sun.jersey.spi.container.ResourceFilters; import io.druid.guice.annotations.Json; import io.druid.guice.annotations.Smile; -import io.druid.java.util.common.Pair; import io.druid.server.coordination.BatchDataSegmentAnnouncer; import io.druid.server.coordination.DataSegmentChangeRequest; import io.druid.server.coordination.SegmentChangeRequestHistory; @@ -245,7 +244,7 @@ public void applyDataSegmentChangeRequests( } final ResponseContext context = createContext(req.getHeader("Accept")); - final ListenableFuture>> future = loadDropRequestHandler + final ListenableFuture> future = loadDropRequestHandler .processBatch(changeRequestList); final AsyncContext asyncContext = req.startAsync(); @@ -281,15 +280,16 @@ public void onStartAsync(AsyncEvent event) throws IOException Futures.addCallback( future, - new FutureCallback>>() + new FutureCallback>() { @Override - public void onSuccess(List> result) + public void onSuccess(List result) { try { HttpServletResponse response = (HttpServletResponse) asyncContext.getResponse(); response.setStatus(HttpServletResponse.SC_OK); - context.inputMapper.writerWithType(HttpLoadQueuePeon.RESPONSE_ENTITY_TYPE_REF).writeValue(asyncContext.getResponse().getOutputStream(), result); + context.inputMapper.writerWithType(HttpLoadQueuePeon.RESPONSE_ENTITY_TYPE_REF) + .writeValue(asyncContext.getResponse().getOutputStream(), result); asyncContext.complete(); } catch (Exception ex) { diff --git a/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java b/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java index 078210485ee2..3efa94f976bc 100644 --- a/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java +++ b/server/src/main/java/io/druid/server/initialization/jetty/JettyServerModule.java @@ -308,6 +308,7 @@ static Server makeAndInitializeServer( @Override public void start() throws Exception { + log.info("Starting Jetty Server..."); server.start(); if (node.isEnableTlsPort()) { // Perform validation @@ -336,6 +337,7 @@ public void start() throws Exception public void stop() { try { + log.info("Stopping Jetty Server..."); server.stop(); } catch (Exception e) { diff --git a/server/src/test/java/io/druid/server/coordination/SegmentLoadDropHandlerTest.java b/server/src/test/java/io/druid/server/coordination/SegmentLoadDropHandlerTest.java index 078152933260..122c83bf07cd 100644 --- a/server/src/test/java/io/druid/server/coordination/SegmentLoadDropHandlerTest.java +++ b/server/src/test/java/io/druid/server/coordination/SegmentLoadDropHandlerTest.java @@ -28,7 +28,6 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Intervals; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.concurrent.ScheduledExecutorFactory; import io.druid.java.util.common.logger.Logger; import io.druid.segment.IndexIO; @@ -221,7 +220,7 @@ public void testSegmentLoading1() throws Exception final DataSegment segment = makeSegment("test", "1", Intervals.of("P1d/2011-04-01")); - segmentLoadDropHandler.removeSegment(segment, DataSegmentChangeCallback.NOOP, true); + segmentLoadDropHandler.removeSegment(segment, DataSegmentChangeCallback.NOOP); Assert.assertFalse(segmentsAnnouncedByMe.contains(segment)); @@ -260,7 +259,7 @@ public void testSegmentLoading2() throws Exception Assert.assertTrue(segmentsAnnouncedByMe.contains(segment)); - segmentLoadDropHandler.removeSegment(segment, DataSegmentChangeCallback.NOOP, true); + segmentLoadDropHandler.removeSegment(segment, DataSegmentChangeCallback.NOOP); Assert.assertFalse(segmentsAnnouncedByMe.contains(segment)); @@ -463,19 +462,24 @@ public void testProcessBatch() throws Exception new SegmentChangeRequestDrop(segment2) ); - ListenableFuture>> future = segmentLoadDropHandler + ListenableFuture> future = segmentLoadDropHandler .processBatch(batch); + List result = future.get(); + Assert.assertEquals(SegmentLoadDropHandler.Status.PENDING, result.get(0).getStatus()); + Assert.assertEquals(SegmentLoadDropHandler.Status.SUCCESS, result.get(1).getStatus()); + for (Runnable runnable : scheduledRunnable) { runnable.run(); } - List> result = future.get(); - Assert.assertEquals(SegmentLoadDropHandler.Status.SUCCESS, result.get(0).rhs); - Assert.assertEquals(SegmentLoadDropHandler.Status.PENDING, result.get(1).rhs); + result = segmentLoadDropHandler.processBatch(batch).get(); + Assert.assertEquals(SegmentLoadDropHandler.Status.SUCCESS, result.get(0).getStatus()); + Assert.assertEquals(SegmentLoadDropHandler.Status.SUCCESS, result.get(1).getStatus()); + - for (Pair p : segmentLoadDropHandler.processBatch(batch).get()) { - Assert.assertEquals(SegmentLoadDropHandler.Status.SUCCESS, p.rhs); + for (SegmentLoadDropHandler.DataSegmentChangeRequestAndStatus e : segmentLoadDropHandler.processBatch(batch).get()) { + Assert.assertEquals(SegmentLoadDropHandler.Status.SUCCESS, e.getStatus()); } segmentLoadDropHandler.stop(); diff --git a/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java index 36e5b72ba45a..ce106ff4fdac 100644 --- a/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java +++ b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java @@ -22,10 +22,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.metamx.emitter.EmittingLogger; +import io.druid.TestUtil; import io.druid.curator.CuratorTestBase; import io.druid.jackson.DefaultObjectMapper; import io.druid.java.util.common.Intervals; import io.druid.segment.IndexIO; +import io.druid.segment.loading.SegmentLoaderConfig; +import io.druid.server.SegmentManager; import io.druid.server.initialization.ZkPathsConfig; import io.druid.server.metrics.NoopServiceEmitter; import io.druid.timeline.DataSegment; @@ -39,6 +42,7 @@ import java.util.Arrays; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledExecutorService; /** */ @@ -97,34 +101,33 @@ public void testLoadDrop() throws Exception CountDownLatch loadLatch = new CountDownLatch(1); CountDownLatch dropLatch = new CountDownLatch(1); - SegmentLoadDropHandler segmentLoadDropHandler = EasyMock.createStrictMock(SegmentLoadDropHandler.class); - - EasyMock.expect( - segmentLoadDropHandler.getSyncLoadingHanlder() - ).andReturn( - new DataSegmentChangeHandler() - { - - @Override - public void addSegment(DataSegment s, DataSegmentChangeCallback callback) - { - if (segment.getIdentifier().equals(s.getIdentifier())) { - loadLatch.countDown(); - callback.execute(); - } - } - - @Override - public void removeSegment(DataSegment s, DataSegmentChangeCallback callback) - { - if (segment.getIdentifier().equals(s.getIdentifier())) { - dropLatch.countDown(); - callback.execute(); - } - } + SegmentLoadDropHandler segmentLoadDropHandler = new SegmentLoadDropHandler( + TestUtil.MAPPER, + new SegmentLoaderConfig(), + EasyMock.createNiceMock(DataSegmentAnnouncer.class), + EasyMock.createNiceMock(DataSegmentServerAnnouncer.class), + EasyMock.createNiceMock(SegmentManager.class), + EasyMock.createNiceMock(ScheduledExecutorService.class) + ) + { + @Override + public void addSegment(DataSegment s, DataSegmentChangeCallback callback) + { + if (segment.getIdentifier().equals(s.getIdentifier())) { + loadLatch.countDown(); + callback.execute(); } - ); - EasyMock.replay(segmentLoadDropHandler); + } + + @Override + public void removeSegment(DataSegment s, DataSegmentChangeCallback callback) + { + if (segment.getIdentifier().equals(s.getIdentifier())) { + dropLatch.countDown(); + callback.execute(); + } + } + }; zkCoordinator = new ZkCoordinator( segmentLoadDropHandler, @@ -156,7 +159,5 @@ public void removeSegment(DataSegment s, DataSegmentChangeCallback callback) } zkCoordinator.stop(); - - EasyMock.verify(segmentLoadDropHandler); } } diff --git a/server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java b/server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java index 60beec89f41f..b9adbb89a017 100644 --- a/server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java +++ b/server/src/test/java/io/druid/server/coordinator/HttpLoadQueuePeonTest.java @@ -32,7 +32,6 @@ import io.druid.discovery.DiscoveryDruidNode; import io.druid.discovery.DruidNodeDiscovery; import io.druid.java.util.common.Intervals; -import io.druid.java.util.common.Pair; import io.druid.java.util.common.RE; import io.druid.server.coordination.DataSegmentChangeRequest; import io.druid.server.coordination.SegmentLoadDropHandler; @@ -195,9 +194,9 @@ public ListenableFuture go( request.getContent().array(), new TypeReference>() {} ); - List> statuses = new ArrayList<>(changeRequests.size()); + List statuses = new ArrayList<>(changeRequests.size()); for (DataSegmentChangeRequest cr : changeRequests) { - statuses.add(Pair.of(cr, SegmentLoadDropHandler.Status.SUCCESS)); + statuses.add(new SegmentLoadDropHandler.DataSegmentChangeRequestAndStatus(cr, SegmentLoadDropHandler.Status.SUCCESS)); } return (ListenableFuture) Futures.immediateFuture(new ByteArrayInputStream(TestUtil.MAPPER.writerWithType( HttpLoadQueuePeon.RESPONSE_ENTITY_TYPE_REF).writeValueAsBytes(statuses)));