From 119e24478d775eb7c87012904072b4e854b5d5b6 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Wed, 3 Jan 2018 08:14:27 -0800 Subject: [PATCH 1/3] Add better logging messages in lookups startup on query nodes --- .../query/lookup/LookupReferencesManager.java | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java index 6200961fad24..7dc88ecbcc6c 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java +++ b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java @@ -65,7 +65,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.LockSupport; import java.util.function.Function; -import java.util.stream.Collectors; /** * This class provide a basic {@link LookupExtractorFactory} references manager. @@ -472,37 +471,30 @@ private List getLookupBeanList(Map lookupBeanList) { - ImmutableMap.Builder builder = ImmutableMap.builder(); - ExecutorService executorService = Execs.multiThreaded( + final ImmutableMap.Builder builder = ImmutableMap.builder(); + final ExecutorService executorService = Execs.multiThreaded( lookupConfig.getNumLookupLoadingThreads(), "LookupReferencesManager-Startup-%s" ); - CompletionService> completionService = + final CompletionService> completionService = new ExecutorCompletionService<>(executorService); try { LOG.info("Starting lookup loading process"); List remainingLookups = lookupBeanList; - for (int i = 0; i < lookupConfig.getLookupStartRetries(); i++) { + for (int i = 0; i < lookupConfig.getLookupStartRetries() && !remainingLookups.isEmpty(); i++) { LOG.info("Round of attempts #%d, [%d] lookups", i + 1, remainingLookups.size()); - Map successfulLookups = + final Map successfulLookups = startLookups(remainingLookups, completionService); builder.putAll(successfulLookups); - List failedLookups = remainingLookups - .stream() - .filter(l -> !successfulLookups.containsKey(l.getName())) - .collect(Collectors.toList()); - if (failedLookups.isEmpty()) { - break; - } else { - // next round - remainingLookups = failedLookups; - } + remainingLookups.removeIf(l -> successfulLookups.containsKey(l.getName())); + } + if (!remainingLookups.isEmpty()) { + LOG.warn( + "Failed to start the following lookups after [%d] attempts: [%s]", + lookupConfig.getLookupStartRetries(), + remainingLookups + ); } - LOG.info( - "Failed to start the following lookups after [%d] attempts: [%s]", - lookupConfig.getLookupStartRetries(), - remainingLookups - ); stateRef.set(new LookupUpdateState(builder.build(), ImmutableList.of(), ImmutableList.of())); } catch (InterruptedException | RuntimeException e) { From dad55eebbd61f6d2216c660b3c1ed598df62ca4f Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Wed, 3 Jan 2018 08:18:18 -0800 Subject: [PATCH 2/3] Make sure list is mutable --- .../java/io/druid/query/lookup/LookupReferencesManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java index 7dc88ecbcc6c..0d11e80b89bc 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java +++ b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java @@ -480,7 +480,7 @@ private void startLookups(final List lookupBeanList) new ExecutorCompletionService<>(executorService); try { LOG.info("Starting lookup loading process"); - List remainingLookups = lookupBeanList; + final List remainingLookups = new ArrayList<>(lookupBeanList); for (int i = 0; i < lookupConfig.getLookupStartRetries() && !remainingLookups.isEmpty(); i++) { LOG.info("Round of attempts #%d, [%d] lookups", i + 1, remainingLookups.size()); final Map successfulLookups = From 6bb2d312734e397b106509800399532dbb0133a8 Mon Sep 17 00:00:00 2001 From: Charles Allen Date: Wed, 3 Jan 2018 08:20:50 -0800 Subject: [PATCH 3/3] Move list to be with other `final` variables --- .../java/io/druid/query/lookup/LookupReferencesManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java index 0d11e80b89bc..df2229f9229b 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java +++ b/server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java @@ -478,9 +478,9 @@ private void startLookups(final List lookupBeanList) ); final CompletionService> completionService = new ExecutorCompletionService<>(executorService); + final List remainingLookups = new ArrayList<>(lookupBeanList); try { LOG.info("Starting lookup loading process"); - final List remainingLookups = new ArrayList<>(lookupBeanList); for (int i = 0; i < lookupConfig.getLookupStartRetries() && !remainingLookups.isEmpty(); i++) { LOG.info("Round of attempts #%d, [%d] lookups", i + 1, remainingLookups.size()); final Map successfulLookups =