From 7cf6db6152499b6e758e4a56e90ab85c992937c1 Mon Sep 17 00:00:00 2001 From: Adam Peck Date: Mon, 22 Aug 2022 17:39:37 -0600 Subject: [PATCH 1/8] Update Curator to 5.3.0 --- docs/configuration/index.md | 17 ---- pom.xml | 2 +- .../apache/druid/curator/CuratorModule.java | 62 +------------ .../apache/druid/curator/ExhibitorConfig.java | 86 ------------------- .../curator/discovery/DiscoveryModule.java | 22 ++--- .../inventory/CuratorInventoryManager.java | 1 - .../druid/curator/CuratorConfigTest.java | 9 -- .../druid/curator/CuratorModuleTest.java | 76 ---------------- .../druid/curator/ExhibitorConfigTest.java | 66 -------------- 9 files changed, 14 insertions(+), 327 deletions(-) delete mode 100644 server/src/main/java/org/apache/druid/curator/ExhibitorConfig.java delete mode 100644 server/src/test/java/org/apache/druid/curator/ExhibitorConfigTest.java diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 5e6f7d7e8657..7fc3a55c372f 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -151,23 +151,6 @@ The following path is used for service discovery. It is **not** affected by `dru |--------|-----------|-------| |`druid.discovery.curator.path`|Services announce themselves under this ZooKeeper path.|`/druid/discovery`| -### Exhibitor - -[Exhibitor](https://github.com/Netflix/exhibitor/wiki) is a supervisor system for ZooKeeper. -Exhibitor can dynamically scale-up/down the cluster of ZooKeeper servers. -Druid can update self-owned list of ZooKeeper servers through Exhibitor without restarting. -That is, it allows Druid to keep the connections of Exhibitor-supervised ZooKeeper servers. - -|Property|Description|Default| -|--------|-----------|-------| -|`druid.exhibitor.service.hosts`|A JSON array which contains the hostnames of Exhibitor instances. Please specify this property if you want to use Exhibitor-supervised cluster.|none| -|`druid.exhibitor.service.port`|The REST port used to connect to Exhibitor.|`8080`| -|`druid.exhibitor.service.restUriPath`|The path of the REST call used to get the server set.|`/exhibitor/v1/cluster/list`| -|`druid.exhibitor.service.useSsl`|Boolean flag for whether or not to use https protocol.|`false`| -|`druid.exhibitor.service.pollingMs`|How often to poll the exhibitors for the list|`10000`| - -Note that `druid.zk.service.host` is used as a backup in case an Exhibitor instance can't be contacted and therefore should still be set. - ### TLS #### General Configuration diff --git a/pom.xml b/pom.xml index ef0b1a5b6604..fb96b8beeec3 100644 --- a/pom.xml +++ b/pom.xml @@ -75,7 +75,7 @@ 8 UTF-8 0.9.0.M2 - 4.3.0 + 5.3.0 3.2.0 2.0.0 2.2.4 diff --git a/server/src/main/java/org/apache/druid/curator/CuratorModule.java b/server/src/main/java/org/apache/druid/curator/CuratorModule.java index a7af6920885a..a7c751e0d2b7 100644 --- a/server/src/main/java/org/apache/druid/curator/CuratorModule.java +++ b/server/src/main/java/org/apache/druid/curator/CuratorModule.java @@ -23,10 +23,6 @@ import com.google.inject.Module; import com.google.inject.Provides; import org.apache.curator.RetryPolicy; -import org.apache.curator.ensemble.EnsembleProvider; -import org.apache.curator.ensemble.exhibitor.DefaultExhibitorRestClient; -import org.apache.curator.ensemble.exhibitor.ExhibitorEnsembleProvider; -import org.apache.curator.ensemble.exhibitor.Exhibitors; import org.apache.curator.ensemble.fixed.FixedEnsembleProvider; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -58,14 +54,13 @@ public void configure(Binder binder) { JsonConfigProvider.bind(binder, CuratorConfig.CONFIG_PREFIX, ZkEnablementConfig.class); JsonConfigProvider.bind(binder, CuratorConfig.CONFIG_PREFIX, CuratorConfig.class); - JsonConfigProvider.bind(binder, ExhibitorConfig.CONFIG_PREFIX, ExhibitorConfig.class); } /** * Create the Curator framework outside of Guice given the ZK config. * Primarily for tests. */ - public static CuratorFramework createCurator(CuratorConfig config, EnsembleProvider ensembleProvider) + public static CuratorFramework createCurator(CuratorConfig config) { final CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder(); if (!Strings.isNullOrEmpty(config.getZkUser()) && !Strings.isNullOrEmpty(config.getZkPwd())) { @@ -78,7 +73,7 @@ public static CuratorFramework createCurator(CuratorConfig config, EnsembleProvi RetryPolicy retryPolicy = new BoundedExponentialBackoffRetry(BASE_SLEEP_TIME_MS, MAX_SLEEP_TIME_MS, MAX_RETRIES); return builder - .ensembleProvider(ensembleProvider) + .ensembleProvider(new FixedEnsembleProvider(config.getZkHosts())) .sessionTimeoutMs(config.getZkSessionTimeoutMs()) .connectionTimeoutMs(config.getZkConnectionTimeoutMs()) .retryPolicy(retryPolicy) @@ -92,13 +87,13 @@ public static CuratorFramework createCurator(CuratorConfig config, EnsembleProvi */ @Provides @LazySingleton - public CuratorFramework makeCurator(ZkEnablementConfig zkEnablementConfig, CuratorConfig config, EnsembleProvider ensembleProvider, Lifecycle lifecycle) + public CuratorFramework makeCurator(ZkEnablementConfig zkEnablementConfig, CuratorConfig config, Lifecycle lifecycle) { if (!zkEnablementConfig.isEnabled()) { throw new RuntimeException("Zookeeper is disabled, cannot create CuratorFramework."); } - final CuratorFramework framework = createCurator(config, ensembleProvider); + final CuratorFramework framework = createCurator(config); framework.getUnhandledErrorListenable().addListener((message, e) -> { log.error(e, "Unhandled error in Curator, stopping server."); @@ -127,55 +122,6 @@ public void stop() return framework; } - /** - * Create an EnsembleProvider given the related configurations. Primarily for tests - * which do not use Guice to do the work. - */ - public static EnsembleProvider createEnsembleProvider(CuratorConfig config, ExhibitorConfig exConfig) - { - if (exConfig.getHosts().isEmpty()) { - return new FixedEnsembleProvider(config.getZkHosts()); - } - - RetryPolicy retryPolicy = new BoundedExponentialBackoffRetry(BASE_SLEEP_TIME_MS, MAX_SLEEP_TIME_MS, MAX_RETRIES); - - return new ExhibitorEnsembleProvider( - new Exhibitors( - exConfig.getHosts(), - exConfig.getRestPort(), - newBackupProvider(config.getZkHosts()) - ), - new DefaultExhibitorRestClient(exConfig.getUseSsl()), - exConfig.getRestUriPath(), - exConfig.getPollingMs(), - retryPolicy - ) - { - @Override - public void start() throws Exception - { - log.debug("Polling the list of ZooKeeper servers for the initial ensemble"); - this.pollForInitialEnsemble(); - super.start(); - } - }; - } - - /** - * Provide an EnsembleProvider via Guice configuration. - */ - @Provides - @LazySingleton - public EnsembleProvider makeEnsembleProvider(CuratorConfig config, ExhibitorConfig exConfig) - { - return createEnsembleProvider(config, exConfig); - } - - private static Exhibitors.BackupConnectionStringProvider newBackupProvider(final String zkHosts) - { - return () -> zkHosts; - } - static class SecuredACLProvider implements ACLProvider { @Override diff --git a/server/src/main/java/org/apache/druid/curator/ExhibitorConfig.java b/server/src/main/java/org/apache/druid/curator/ExhibitorConfig.java deleted file mode 100644 index 97e7bb74df5e..000000000000 --- a/server/src/main/java/org/apache/druid/curator/ExhibitorConfig.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.curator; - -import com.fasterxml.jackson.annotation.JsonProperty; - -import javax.validation.constraints.Max; -import javax.validation.constraints.Min; -import java.util.ArrayList; -import java.util.List; - -/** - */ -public class ExhibitorConfig -{ - static final String CONFIG_PREFIX = "druid.exhibitor.service"; - - @JsonProperty - private List hosts = new ArrayList<>(); - - @JsonProperty("port") - @Min(0) - @Max(0xffff) - private int restPort = 8080; - - @JsonProperty - private String restUriPath = "/exhibitor/v1/cluster/list"; - - @JsonProperty - private boolean useSsl = false; - - @JsonProperty - @Min(0) - private int pollingMs = 10000; - - public static ExhibitorConfig create(List hosts) - { - ExhibitorConfig config = new ExhibitorConfig(); - if (hosts != null) { - config.hosts = hosts; - } - return config; - } - - public List getHosts() - { - return hosts; - } - - public int getRestPort() - { - return restPort; - } - - public String getRestUriPath() - { - return restUriPath; - } - - public boolean getUseSsl() - { - return useSsl; - } - - public int getPollingMs() - { - return pollingMs; - } -} diff --git a/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java b/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java index 384b4d8832e2..508d72891dcb 100644 --- a/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java +++ b/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java @@ -31,7 +31,6 @@ import com.google.inject.name.Named; import com.google.inject.name.Names; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.utils.CloseableExecutorService; import org.apache.curator.utils.ZKPaths; import org.apache.curator.x.discovery.DownInstancePolicy; import org.apache.curator.x.discovery.InstanceFilter; @@ -69,6 +68,7 @@ import java.util.List; import java.util.Properties; import java.util.Set; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; @@ -416,14 +416,10 @@ public ServiceCacheBuilder executorService(ExecutorService executorService) return this; } - @Override - public ServiceCacheBuilder executorService(CloseableExecutorService closeableExecutorService) - { - return this; - } - private static class NoopServiceCache implements ServiceCache { + private static final CountDownLatch COUNT_DOWN_LATCH = new CountDownLatch(0); + @Override public List> getInstances() { @@ -436,6 +432,12 @@ public void start() // nothing } + @Override + public CountDownLatch startImmediate() throws Exception + { + return COUNT_DOWN_LATCH; + } + @Override public void close() { @@ -505,12 +507,6 @@ public ServiceProviderBuilder executorService(ExecutorService executorService { return this; } - - @Override - public ServiceProviderBuilder executorService(CloseableExecutorService closeableExecutorService) - { - return this; - } } private static class NoopServiceProvider implements ServiceProvider diff --git a/server/src/main/java/org/apache/druid/curator/inventory/CuratorInventoryManager.java b/server/src/main/java/org/apache/druid/curator/inventory/CuratorInventoryManager.java index 5df07e962490..4bceb2a8594d 100644 --- a/server/src/main/java/org/apache/druid/curator/inventory/CuratorInventoryManager.java +++ b/server/src/main/java/org/apache/druid/curator/inventory/CuratorInventoryManager.java @@ -278,7 +278,6 @@ public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) th // This close() call actually calls shutdownNow() on the executor registered with the Cache object, it // better have its own executor or ignore shutdownNow() calls... log.debug("Closing inventory cache for %s. Also removing listeners.", containerKey); - removed.getCache().getListenable().clear(); removed.getCache().close(); strategy.deadContainer(removed.getContainer()); diff --git a/server/src/test/java/org/apache/druid/curator/CuratorConfigTest.java b/server/src/test/java/org/apache/druid/curator/CuratorConfigTest.java index ea0617dddaa6..b3719d85d1fc 100644 --- a/server/src/test/java/org/apache/druid/curator/CuratorConfigTest.java +++ b/server/src/test/java/org/apache/druid/curator/CuratorConfigTest.java @@ -23,8 +23,6 @@ import org.junit.Assert; import org.junit.Test; -import java.util.Arrays; - public class CuratorConfigTest extends JsonConfigTesterBase { @Test @@ -54,11 +52,4 @@ public void testCreate() Assert.assertNull(config.getZkUser()); Assert.assertEquals("digest", config.getAuthScheme()); } - - @Test - public void testExhibitorCreate() - { - ExhibitorConfig config = ExhibitorConfig.create(Arrays.asList("foo:2181", "bar:2181")); - Assert.assertEquals(2, config.getHosts().size()); - } } diff --git a/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java b/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java index 24264b219d0d..ddd7ddbe1767 100644 --- a/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java +++ b/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java @@ -21,9 +21,6 @@ import com.google.inject.Injector; import org.apache.curator.RetryPolicy; -import org.apache.curator.ensemble.EnsembleProvider; -import org.apache.curator.ensemble.exhibitor.ExhibitorEnsembleProvider; -import org.apache.curator.ensemble.fixed.FixedEnsembleProvider; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.retry.BoundedExponentialBackoffRetry; import org.apache.curator.retry.ExponentialBackoffRetry; @@ -44,10 +41,8 @@ public final class CuratorModuleTest { - private static final String CURATOR_HOST_KEY = CuratorConfig.CONFIG_PREFIX + "." + CuratorConfig.HOST; private static final String CURATOR_CONNECTION_TIMEOUT_MS_KEY = CuratorConfig.CONFIG_PREFIX + "." + CuratorConfig.CONNECTION_TIMEOUT_MS; - private static final String EXHIBITOR_HOSTS_KEY = ExhibitorConfig.CONFIG_PREFIX + ".hosts"; @Rule public final ExpectedSystemExit exit = ExpectedSystemExit.none(); @@ -55,77 +50,6 @@ public final class CuratorModuleTest @Rule public final LoggerCaptureRule logger = new LoggerCaptureRule(CuratorModule.class); - @Test - public void defaultEnsembleProvider() - { - Injector injector = newInjector(new Properties()); - injector.getInstance(CuratorFramework.class); // initialize related components - EnsembleProvider ensembleProvider = injector.getInstance(EnsembleProvider.class); - Assert.assertTrue( - "EnsembleProvider should be FixedEnsembleProvider", - ensembleProvider instanceof FixedEnsembleProvider - ); - Assert.assertEquals( - "The connectionString should be 'localhost'", - "localhost", ensembleProvider.getConnectionString() - ); - } - - @Test - public void fixedZkHosts() - { - Properties props = new Properties(); - props.setProperty(CURATOR_HOST_KEY, "hostA"); - Injector injector = newInjector(props); - - injector.getInstance(CuratorFramework.class); // initialize related components - EnsembleProvider ensembleProvider = injector.getInstance(EnsembleProvider.class); - Assert.assertTrue( - "EnsembleProvider should be FixedEnsembleProvider", - ensembleProvider instanceof FixedEnsembleProvider - ); - Assert.assertEquals( - "The connectionString should be 'hostA'", - "hostA", ensembleProvider.getConnectionString() - ); - } - - @Test - public void exhibitorEnsembleProvider() - { - Properties props = new Properties(); - props.setProperty(CURATOR_HOST_KEY, "hostA"); - props.setProperty(EXHIBITOR_HOSTS_KEY, "[\"hostB\"]"); - Injector injector = newInjector(props); - - injector.getInstance(CuratorFramework.class); // initialize related components - EnsembleProvider ensembleProvider = injector.getInstance(EnsembleProvider.class); - Assert.assertTrue( - "EnsembleProvider should be ExhibitorEnsembleProvider", - ensembleProvider instanceof ExhibitorEnsembleProvider - ); - } - - @Test - public void emptyExhibitorHosts() - { - Properties props = new Properties(); - props.setProperty(CURATOR_HOST_KEY, "hostB"); - props.setProperty(EXHIBITOR_HOSTS_KEY, "[]"); - Injector injector = newInjector(props); - - injector.getInstance(CuratorFramework.class); // initialize related components - EnsembleProvider ensembleProvider = injector.getInstance(EnsembleProvider.class); - Assert.assertTrue( - "EnsembleProvider should be FixedEnsembleProvider", - ensembleProvider instanceof FixedEnsembleProvider - ); - Assert.assertEquals( - "The connectionString should be 'hostB'", - "hostB", ensembleProvider.getConnectionString() - ); - } - @Test public void exitsJvmWhenMaxRetriesExceeded() throws Exception { diff --git a/server/src/test/java/org/apache/druid/curator/ExhibitorConfigTest.java b/server/src/test/java/org/apache/druid/curator/ExhibitorConfigTest.java deleted file mode 100644 index b8b711f926c8..000000000000 --- a/server/src/test/java/org/apache/druid/curator/ExhibitorConfigTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.curator; - -import org.apache.druid.guice.JsonConfigTesterBase; -import org.junit.Assert; -import org.junit.Test; - -import java.util.List; -import java.util.Properties; - -public class ExhibitorConfigTest extends JsonConfigTesterBase -{ - @Test - public void testSerde() - { - propertyValues.put(getPropertyKey("hosts"), "[\"hostA\",\"hostB\"]"); - propertyValues.put(getPropertyKey("port"), "80"); - propertyValues.put(getPropertyKey("restUriPath"), "/list"); - propertyValues.put(getPropertyKey("useSsl"), "true"); - propertyValues.put(getPropertyKey("pollingMs"), "1000"); - testProperties.putAll(propertyValues); - configProvider.inject(testProperties, configurator); - ExhibitorConfig config = configProvider.get().get(); - - List hosts = config.getHosts(); - Assert.assertEquals(2, hosts.size()); - Assert.assertTrue(hosts.contains("hostA")); - Assert.assertTrue(hosts.contains("hostB")); - Assert.assertEquals(80, config.getRestPort()); - Assert.assertEquals("/list", config.getRestUriPath()); - Assert.assertTrue(config.getUseSsl()); - Assert.assertEquals(1000, config.getPollingMs()); - } - - @Test - public void defaultValues() - { - configProvider.inject(new Properties(), configurator); - ExhibitorConfig config = configProvider.get().get(); - - List hosts = config.getHosts(); - Assert.assertTrue(hosts.isEmpty()); - Assert.assertEquals(8080, config.getRestPort()); - Assert.assertEquals("/exhibitor/v1/cluster/list", config.getRestUriPath()); - Assert.assertFalse(config.getUseSsl()); - Assert.assertEquals(10000, config.getPollingMs()); - } -} From 0bc59600c378a577ec91eef0301ae3166bb13378 Mon Sep 17 00:00:00 2001 From: Adam Peck Date: Mon, 22 Aug 2022 19:29:16 -0600 Subject: [PATCH 2/8] Update licenses.yaml --- licenses.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/licenses.yaml b/licenses.yaml index 22c02b5241bc..3052d62392c6 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -1662,7 +1662,7 @@ name: Apache Curator license_category: binary module: java-core license_name: Apache License version 2.0 -version: 4.3.0 +version: 5.3.0 libraries: - org.apache.curator: curator-client - org.apache.curator: curator-framework @@ -1671,16 +1671,16 @@ libraries: notices: - curator-client: | Curator Client - Copyright 2011-2018 The Apache Software Foundation + Copyright 2011-2022 The Apache Software Foundation - curator-framework: | Curator Framework - Copyright 2011-2018 The Apache Software Foundation + Copyright 2011-2022 The Apache Software Foundation - curator-recipes: | Curator Recipes - Copyright 2011-2018 The Apache Software Foundation + Copyright 2011-2022 The Apache Software Foundation - curator-x-discovery: | Curator Service Discovery - Copyright 2011-2018 The Apache Software Foundation + Copyright 2011-2022 The Apache Software Foundation --- From c35dbdb2a78bdce816ba423e481b16ca580c3281 Mon Sep 17 00:00:00 2001 From: Adam Peck Date: Tue, 23 Aug 2022 00:04:18 -0600 Subject: [PATCH 3/8] Fix inspections + add tests. --- .../apache/druid/curator/CuratorModule.java | 4 ++-- .../curator/discovery/DiscoveryModule.java | 6 ++---- .../druid/curator/CuratorModuleTest.java | 20 +++++++++++++++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/apache/druid/curator/CuratorModule.java b/server/src/main/java/org/apache/druid/curator/CuratorModule.java index a7c751e0d2b7..665024fd7716 100644 --- a/server/src/main/java/org/apache/druid/curator/CuratorModule.java +++ b/server/src/main/java/org/apache/druid/curator/CuratorModule.java @@ -45,8 +45,8 @@ public class CuratorModule implements Module { private static final Logger log = new Logger(CuratorModule.class); - private static final int BASE_SLEEP_TIME_MS = 1000; - private static final int MAX_SLEEP_TIME_MS = 45000; + static final int BASE_SLEEP_TIME_MS = 1000; + static final int MAX_SLEEP_TIME_MS = 45000; private static final int MAX_RETRIES = 29; @Override diff --git a/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java b/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java index 508d72891dcb..4889261fcae4 100644 --- a/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java +++ b/server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java @@ -418,8 +418,6 @@ public ServiceCacheBuilder executorService(ExecutorService executorService) private static class NoopServiceCache implements ServiceCache { - private static final CountDownLatch COUNT_DOWN_LATCH = new CountDownLatch(0); - @Override public List> getInstances() { @@ -433,9 +431,9 @@ public void start() } @Override - public CountDownLatch startImmediate() throws Exception + public CountDownLatch startImmediate() { - return COUNT_DOWN_LATCH; + return null; } @Override diff --git a/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java b/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java index ddd7ddbe1767..d4023d4dd8b7 100644 --- a/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java +++ b/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java @@ -20,6 +20,8 @@ package org.apache.druid.curator; import com.google.inject.Injector; + +import org.apache.curator.CuratorZookeeperClient; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.retry.BoundedExponentialBackoffRetry; @@ -30,6 +32,8 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.LogEvent; import org.hamcrest.CoreMatchers; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; import org.junit.Assert; import org.junit.Ignore; import org.junit.Rule; @@ -50,6 +54,22 @@ public final class CuratorModuleTest @Rule public final LoggerCaptureRule logger = new LoggerCaptureRule(CuratorModule.class); + @Test + public void createsCuratorFrameworkAsConfigured() throws Exception + { + CuratorConfig config = CuratorConfig.create("myhost1:2888,myhost2:2888"); + CuratorFramework curatorFramework = CuratorModule.createCurator(config); + CuratorZookeeperClient client = curatorFramework.getZookeeperClient(); + + Assert.assertEquals(config.getZkHosts(), client.getCurrentConnectionString()); + Assert.assertEquals(config.getZkConnectionTimeoutMs(), client.getConnectionTimeoutMs()); + + MatcherAssert.assertThat(client.getRetryPolicy(), Matchers.instanceOf(BoundedExponentialBackoffRetry.class)); + BoundedExponentialBackoffRetry retryPolicy = (BoundedExponentialBackoffRetry) client.getRetryPolicy(); + Assert.assertEquals(CuratorModule.BASE_SLEEP_TIME_MS, retryPolicy.getBaseSleepTimeMs()); + Assert.assertEquals(CuratorModule.MAX_SLEEP_TIME_MS, retryPolicy.getMaxSleepTimeMs()); + } + @Test public void exitsJvmWhenMaxRetriesExceeded() throws Exception { From 517bb065e5879f8d7837ded6da1eec520313cf08 Mon Sep 17 00:00:00 2001 From: Adam Peck Date: Tue, 23 Aug 2022 09:22:17 -0600 Subject: [PATCH 4/8] Fix checkstyle --- .../test/java/org/apache/druid/curator/CuratorModuleTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java b/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java index d4023d4dd8b7..2c5d94865e48 100644 --- a/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java +++ b/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java @@ -20,7 +20,6 @@ package org.apache.druid.curator; import com.google.inject.Injector; - import org.apache.curator.CuratorZookeeperClient; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; From 7935d8ab21077f1eadcf379c3c3f0d7bd4a160b5 Mon Sep 17 00:00:00 2001 From: Adam Peck Date: Tue, 23 Aug 2022 16:55:17 -0600 Subject: [PATCH 5/8] Another intellij inspection fix --- .../test/java/org/apache/druid/curator/CuratorModuleTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java b/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java index 2c5d94865e48..20b86dc59660 100644 --- a/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java +++ b/server/src/test/java/org/apache/druid/curator/CuratorModuleTest.java @@ -54,7 +54,7 @@ public final class CuratorModuleTest public final LoggerCaptureRule logger = new LoggerCaptureRule(CuratorModule.class); @Test - public void createsCuratorFrameworkAsConfigured() throws Exception + public void createsCuratorFrameworkAsConfigured() { CuratorConfig config = CuratorConfig.create("myhost1:2888,myhost2:2888"); CuratorFramework curatorFramework = CuratorModule.createCurator(config); From 5608fbc622a2f4e11c81089758274e8b15446590 Mon Sep 17 00:00:00 2001 From: Adam Peck Date: Tue, 23 Aug 2022 16:59:30 -0600 Subject: [PATCH 6/8] Update curator exclusions --- pom.xml | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/pom.xml b/pom.xml index fb96b8beeec3..ecb8b212aa7f 100644 --- a/pom.xml +++ b/pom.xml @@ -346,34 +346,42 @@ org.apache.curator curator-client ${apache.curator.version} + + + org.slf4j + slf4j-api + + + com.google.guava + guava + + + org.apache.zookeeper + zookeeper + + org.apache.curator curator-framework ${apache.curator.version} - - - org.jboss.netty - netty - - org.apache.curator curator-recipes ${apache.curator.version} + + + org.apache.curator + curator-x-discovery + ${apache.curator.version} - org.apache.zookeeper - zookeeper + com.fasterxml.jackson.core + jackson-databind - - org.apache.curator - curator-x-discovery - ${apache.curator.version} - org.apache.calcite calcite-core @@ -984,8 +992,8 @@ ${apache.curator.version} - org.jboss.netty - netty + org.junit.jupiter + junit-jupiter-api test From c6c4dcb458f4e8d4364707fd590b48a7cd15c536 Mon Sep 17 00:00:00 2001 From: Adam Peck Date: Thu, 25 Aug 2022 13:30:47 -0600 Subject: [PATCH 7/8] Cleanup new exhibitor references --- .../org/apache/druid/testsEx/cluster/ZooKeeperClient.java | 6 +----- .../org/apache/druid/testsEx/config/ResolvedConfig.java | 7 ------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/cluster/ZooKeeperClient.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/cluster/ZooKeeperClient.java index cad00d1ae4ab..32f6cb1fdfce 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/cluster/ZooKeeperClient.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/cluster/ZooKeeperClient.java @@ -19,11 +19,9 @@ package org.apache.druid.testsEx.cluster; -import org.apache.curator.ensemble.EnsembleProvider; import org.apache.curator.framework.CuratorFramework; import org.apache.druid.curator.CuratorConfig; import org.apache.druid.curator.CuratorModule; -import org.apache.druid.curator.ExhibitorConfig; import org.apache.druid.java.util.common.ISE; import org.apache.druid.testsEx.config.ResolvedConfig; import org.apache.druid.testsEx.config.ResolvedService.ResolvedZk; @@ -56,9 +54,7 @@ public ZooKeeperClient(ResolvedConfig config) private void prepare() { CuratorConfig curatorConfig = clusterConfig.toCuratorConfig(); - ExhibitorConfig exhibitorConfig = clusterConfig.toExhibitorConfig(); - EnsembleProvider ensembleProvider = CuratorModule.createEnsembleProvider(curatorConfig, exhibitorConfig); - curatorFramework = CuratorModule.createCurator(curatorConfig, ensembleProvider); + curatorFramework = CuratorModule.createCurator(curatorConfig); } private void awaitReady() diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java index 6bdfe96b2f93..5be5c839a9a0 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java @@ -22,7 +22,6 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import org.apache.druid.curator.CuratorConfig; -import org.apache.druid.curator.ExhibitorConfig; import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.ISE; import org.apache.druid.testing.IntegrationTestingConfigProvider; @@ -274,12 +273,6 @@ public CuratorConfig toCuratorConfig() return CuratorConfig.create(zk.clientHosts()); } - public ExhibitorConfig toExhibitorConfig() - { - // Does not yet support exhibitors - return ExhibitorConfig.create(Collections.emptyList()); - } - /** * Map from old-style config file (and settings) name to the * corresponding property. From cbd4e15fbb77e401fe6dee8a2a50ed55c6f733db Mon Sep 17 00:00:00 2001 From: Adam Peck Date: Fri, 26 Aug 2022 12:29:56 -0600 Subject: [PATCH 8/8] Remove unused dep and checkstyle fix --- integration-tests-ex/cases/pom.xml | 4 ---- .../java/org/apache/druid/testsEx/config/ResolvedConfig.java | 1 - 2 files changed, 5 deletions(-) diff --git a/integration-tests-ex/cases/pom.xml b/integration-tests-ex/cases/pom.xml index 5c34af881d68..98483d708dc1 100644 --- a/integration-tests-ex/cases/pom.xml +++ b/integration-tests-ex/cases/pom.xml @@ -91,10 +91,6 @@ org.apache.curator curator-framework - - org.apache.curator - curator-client - com.google.guava guava diff --git a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java index 5be5c839a9a0..bdfc17dac11b 100644 --- a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java +++ b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/config/ResolvedConfig.java @@ -36,7 +36,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry;