Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 0 additions & 17 deletions docs/configuration/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -151,23 +151,6 @@ The following path is used for service discovery. It is **not** affected by `dru
|--------|-----------|-------|
|`druid.discovery.curator.path`|Services announce themselves under this ZooKeeper path.|`/druid/discovery`|

### Exhibitor

[Exhibitor](https://github.com/Netflix/exhibitor/wiki) is a supervisor system for ZooKeeper.
Exhibitor can dynamically scale-up/down the cluster of ZooKeeper servers.
Druid can update self-owned list of ZooKeeper servers through Exhibitor without restarting.
That is, it allows Druid to keep the connections of Exhibitor-supervised ZooKeeper servers.

|Property|Description|Default|
|--------|-----------|-------|
|`druid.exhibitor.service.hosts`|A JSON array which contains the hostnames of Exhibitor instances. Please specify this property if you want to use Exhibitor-supervised cluster.|none|
|`druid.exhibitor.service.port`|The REST port used to connect to Exhibitor.|`8080`|
|`druid.exhibitor.service.restUriPath`|The path of the REST call used to get the server set.|`/exhibitor/v1/cluster/list`|
|`druid.exhibitor.service.useSsl`|Boolean flag for whether or not to use https protocol.|`false`|
|`druid.exhibitor.service.pollingMs`|How often to poll the exhibitors for the list|`10000`|

Note that `druid.zk.service.host` is used as a backup in case an Exhibitor instance can't be contacted and therefore should still be set.

### TLS

#### General Configuration
Expand Down
4 changes: 0 additions & 4 deletions integration-tests-ex/cases/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,6 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,9 @@

package org.apache.druid.testsEx.cluster;

import org.apache.curator.ensemble.EnsembleProvider;
import org.apache.curator.framework.CuratorFramework;
import org.apache.druid.curator.CuratorConfig;
import org.apache.druid.curator.CuratorModule;
import org.apache.druid.curator.ExhibitorConfig;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.testsEx.config.ResolvedConfig;
import org.apache.druid.testsEx.config.ResolvedService.ResolvedZk;
Expand Down Expand Up @@ -56,9 +54,7 @@ public ZooKeeperClient(ResolvedConfig config)
private void prepare()
{
CuratorConfig curatorConfig = clusterConfig.toCuratorConfig();
ExhibitorConfig exhibitorConfig = clusterConfig.toExhibitorConfig();
EnsembleProvider ensembleProvider = CuratorModule.createEnsembleProvider(curatorConfig, exhibitorConfig);
curatorFramework = CuratorModule.createCurator(curatorConfig, ensembleProvider);
curatorFramework = CuratorModule.createCurator(curatorConfig);
}

private void awaitReady()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import org.apache.druid.curator.CuratorConfig;
import org.apache.druid.curator.ExhibitorConfig;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.testing.IntegrationTestingConfigProvider;
Expand All @@ -37,7 +36,6 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
Expand Down Expand Up @@ -274,12 +272,6 @@ public CuratorConfig toCuratorConfig()
return CuratorConfig.create(zk.clientHosts());
}

public ExhibitorConfig toExhibitorConfig()
{
// Does not yet support exhibitors
return ExhibitorConfig.create(Collections.emptyList());
}

/**
* Map from old-style config file (and settings) name to the
* corresponding property.
Expand Down
10 changes: 5 additions & 5 deletions licenses.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1662,7 +1662,7 @@ name: Apache Curator
license_category: binary
module: java-core
license_name: Apache License version 2.0
version: 4.3.0
version: 5.3.0
libraries:
- org.apache.curator: curator-client
- org.apache.curator: curator-framework
Expand All @@ -1671,16 +1671,16 @@ libraries:
notices:
- curator-client: |
Curator Client
Copyright 2011-2018 The Apache Software Foundation
Copyright 2011-2022 The Apache Software Foundation
- curator-framework: |
Curator Framework
Copyright 2011-2018 The Apache Software Foundation
Copyright 2011-2022 The Apache Software Foundation
- curator-recipes: |
Curator Recipes
Copyright 2011-2018 The Apache Software Foundation
Copyright 2011-2022 The Apache Software Foundation
- curator-x-discovery: |
Curator Service Discovery
Copyright 2011-2018 The Apache Software Foundation
Copyright 2011-2022 The Apache Software Foundation

---

Expand Down
40 changes: 24 additions & 16 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@
<java.version>8</java.version>
<project.build.resourceEncoding>UTF-8</project.build.resourceEncoding>
<aether.version>0.9.0.M2</aether.version>
<apache.curator.version>4.3.0</apache.curator.version>
<apache.curator.version>5.3.0</apache.curator.version>
<apache.kafka.version>3.2.0</apache.kafka.version>
<apache.ranger.version>2.0.0</apache.ranger.version>
<apache.ranger.gson.version>2.2.4</apache.ranger.gson.version>
Expand Down Expand Up @@ -348,34 +348,42 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-client</artifactId>
<version>${apache.curator.version}</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>${apache.curator.version}</version>
<exclusions>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>${apache.curator.version}</version>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-x-discovery</artifactId>
<version>${apache.curator.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-x-discovery</artifactId>
<version>${apache.curator.version}</version>
</dependency>
<dependency>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
Expand Down Expand Up @@ -986,8 +994,8 @@
<version>${apache.curator.version}</version>
<exclusions>
<exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
</exclusion>
</exclusions>
<scope>test</scope>
Expand Down
66 changes: 6 additions & 60 deletions server/src/main/java/org/apache/druid/curator/CuratorModule.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,6 @@
import com.google.inject.Module;
import com.google.inject.Provides;
import org.apache.curator.RetryPolicy;
import org.apache.curator.ensemble.EnsembleProvider;
import org.apache.curator.ensemble.exhibitor.DefaultExhibitorRestClient;
import org.apache.curator.ensemble.exhibitor.ExhibitorEnsembleProvider;
import org.apache.curator.ensemble.exhibitor.Exhibitors;
import org.apache.curator.ensemble.fixed.FixedEnsembleProvider;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
Expand All @@ -49,23 +45,22 @@ public class CuratorModule implements Module
{
private static final Logger log = new Logger(CuratorModule.class);

private static final int BASE_SLEEP_TIME_MS = 1000;
private static final int MAX_SLEEP_TIME_MS = 45000;
static final int BASE_SLEEP_TIME_MS = 1000;
static final int MAX_SLEEP_TIME_MS = 45000;
private static final int MAX_RETRIES = 29;

@Override
public void configure(Binder binder)
{
JsonConfigProvider.bind(binder, CuratorConfig.CONFIG_PREFIX, ZkEnablementConfig.class);
JsonConfigProvider.bind(binder, CuratorConfig.CONFIG_PREFIX, CuratorConfig.class);
JsonConfigProvider.bind(binder, ExhibitorConfig.CONFIG_PREFIX, ExhibitorConfig.class);
}

/**
* Create the Curator framework outside of Guice given the ZK config.
* Primarily for tests.
*/
public static CuratorFramework createCurator(CuratorConfig config, EnsembleProvider ensembleProvider)
public static CuratorFramework createCurator(CuratorConfig config)
{
final CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
if (!Strings.isNullOrEmpty(config.getZkUser()) && !Strings.isNullOrEmpty(config.getZkPwd())) {
Expand All @@ -78,7 +73,7 @@ public static CuratorFramework createCurator(CuratorConfig config, EnsembleProvi
RetryPolicy retryPolicy = new BoundedExponentialBackoffRetry(BASE_SLEEP_TIME_MS, MAX_SLEEP_TIME_MS, MAX_RETRIES);

return builder
.ensembleProvider(ensembleProvider)
.ensembleProvider(new FixedEnsembleProvider(config.getZkHosts()))
.sessionTimeoutMs(config.getZkSessionTimeoutMs())
.connectionTimeoutMs(config.getZkConnectionTimeoutMs())
.retryPolicy(retryPolicy)
Expand All @@ -92,13 +87,13 @@ public static CuratorFramework createCurator(CuratorConfig config, EnsembleProvi
*/
@Provides
@LazySingleton
public CuratorFramework makeCurator(ZkEnablementConfig zkEnablementConfig, CuratorConfig config, EnsembleProvider ensembleProvider, Lifecycle lifecycle)
public CuratorFramework makeCurator(ZkEnablementConfig zkEnablementConfig, CuratorConfig config, Lifecycle lifecycle)
{
if (!zkEnablementConfig.isEnabled()) {
throw new RuntimeException("Zookeeper is disabled, cannot create CuratorFramework.");
}

final CuratorFramework framework = createCurator(config, ensembleProvider);
final CuratorFramework framework = createCurator(config);

framework.getUnhandledErrorListenable().addListener((message, e) -> {
log.error(e, "Unhandled error in Curator, stopping server.");
Expand Down Expand Up @@ -127,55 +122,6 @@ public void stop()
return framework;
}

/**
* Create an EnsembleProvider given the related configurations. Primarily for tests
* which do not use Guice to do the work.
*/
public static EnsembleProvider createEnsembleProvider(CuratorConfig config, ExhibitorConfig exConfig)
{
if (exConfig.getHosts().isEmpty()) {
return new FixedEnsembleProvider(config.getZkHosts());
}

RetryPolicy retryPolicy = new BoundedExponentialBackoffRetry(BASE_SLEEP_TIME_MS, MAX_SLEEP_TIME_MS, MAX_RETRIES);

return new ExhibitorEnsembleProvider(
new Exhibitors(
exConfig.getHosts(),
exConfig.getRestPort(),
newBackupProvider(config.getZkHosts())
),
new DefaultExhibitorRestClient(exConfig.getUseSsl()),
exConfig.getRestUriPath(),
exConfig.getPollingMs(),
retryPolicy
)
{
@Override
public void start() throws Exception
{
log.debug("Polling the list of ZooKeeper servers for the initial ensemble");
this.pollForInitialEnsemble();
super.start();
}
};
}

/**
* Provide an EnsembleProvider via Guice configuration.
*/
@Provides
@LazySingleton
public EnsembleProvider makeEnsembleProvider(CuratorConfig config, ExhibitorConfig exConfig)
{
return createEnsembleProvider(config, exConfig);
}

private static Exhibitors.BackupConnectionStringProvider newBackupProvider(final String zkHosts)
{
return () -> zkHosts;
}

static class SecuredACLProvider implements ACLProvider
{
@Override
Expand Down
86 changes: 0 additions & 86 deletions server/src/main/java/org/apache/druid/curator/ExhibitorConfig.java

This file was deleted.

Loading