From f8be8657e649d0490e9ed1f1ef52234b3c31435e Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Sun, 23 Aug 2015 14:55:11 +0100 Subject: [PATCH 01/25] KAFKA-1387: First cut, node dependency on curator --- .../consumer/ZookeeperConsumerConnector.scala | 19 +- core/src/main/scala/kafka/utils/ZkUtils.scala | 168 +++++++++++++++++- 2 files changed, 180 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index e42d10488f8df..9646b62f03698 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -34,9 +34,10 @@ import kafka.network.BlockingChannel import kafka.serializer._ import kafka.utils.CoreUtils.inLock import kafka.utils.ZkUtils._ +import kafka.utils.ZKWatchedEphemeral import kafka.utils._ import org.I0Itec.zkclient.exception.ZkNodeExistsException -import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient} +import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection} import org.apache.zookeeper.Watcher.Event.KeeperState import scala.collection._ @@ -90,6 +91,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private val rebalanceLock = new Object private var fetcher: Option[ConsumerFetcherManager] = None private var zkClient: ZkClient = null + private var zkConnection : ZkConnection = null private var topicRegistry = new Pool[String, Pool[Int, PartitionTopicInfo]] private val checkpointedZkOffsets = new Pool[TopicAndPartition, Long] private val topicThreadIdAndQueues = new Pool[(String, ConsumerThreadId), BlockingQueue[FetchedDataChunk]] @@ -178,7 +180,9 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private def connectZk() { info("Connecting to zookeeper instance at " + config.zkConnect) - zkClient = ZkUtils.createZkClient(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + var (client, connection) = ZkUtils.createZkClientAndConnection(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + zkClient = client + zkConnection = connection } // Blocks until the offset manager is located and a channel is established to it. @@ -255,15 +259,20 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, // this API is used by unit tests only def getTopicRegistry: Pool[String, Pool[Int, PartitionTopicInfo]] = topicRegistry - + private var zkWatchedEphemeral : ZKWatchedEphemeral = null private def registerConsumerInZK(dirs: ZKGroupDirs, consumerIdString: String, topicCount: TopicCount) { info("begin registering consumer " + consumerIdString + " in ZK") val timestamp = SystemTime.milliseconds.toString val consumerRegistrationInfo = Json.encode(Map("version" -> 1, "subscription" -> topicCount.getTopicCountMap, "pattern" -> topicCount.pattern, "timestamp" -> timestamp)) - createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, - (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) + //createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, + // (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) + + if(zkWatchedEphemeral != null) + zkWatchedEphemeral.halt + zkWatchedEphemeral = new ZKWatchedEphemeral(dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, zkConnection) + zkWatchedEphemeral.createAndWatch info("end registering consumer " + consumerIdString + " in ZK") } diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 74b587e04cdd6..2aa900631aa9b 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -20,7 +20,7 @@ package kafka.utils import kafka.cluster._ import kafka.consumer.{ConsumerThreadId, TopicCount} import kafka.server.ConfigType -import org.I0Itec.zkclient.ZkClient +import org.I0Itec.zkclient.{ZkClient,ZkConnection} import org.I0Itec.zkclient.exception.{ZkNodeExistsException, ZkNoNodeException, ZkMarshallingError, ZkBadVersionException} import org.I0Itec.zkclient.serialize.ZkSerializer @@ -36,6 +36,15 @@ import kafka.controller.KafkaController import kafka.controller.LeaderIsrAndControllerEpoch import kafka.common.TopicAndPartition +import org.apache.zookeeper.AsyncCallback.StatCallback +import org.apache.zookeeper.AsyncCallback.StringCallback +import org.apache.zookeeper.CreateMode +import org.apache.zookeeper.KeeperException.Code +import org.apache.zookeeper.WatchedEvent +import org.apache.zookeeper.Watcher +import org.apache.zookeeper.ZooDefs.Ids +import org.apache.zookeeper.ZooKeeper + object ZkUtils extends Logging { val ConsumersPath = "/consumers" val BrokerIdsPath = "/brokers/ids" @@ -809,7 +818,13 @@ object ZkUtils extends Logging { def createZkClient(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): ZkClient = { val zkClient = new ZkClient(zkUrl, sessionTimeout, connectionTimeout, ZKStringSerializer) - zkClient + zkClient + } + + def createZkClientAndConnection(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): (ZkClient, ZkConnection) = { + val zkConnection = new ZkConnection(zkUrl, sessionTimeout) + val zkClient = new ZkClient(zkConnection, connectionTimeout, ZKStringSerializer) + (zkClient, zkConnection) } } @@ -892,3 +907,152 @@ object ZkPath { client.createPersistentSequential(path, data) } } + +class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConnection : ZkConnection) extends Logging { + private val path = ephemeralPath + private val data = ephemeralData + private val zkHandle = zkConnection.getZookeeper + private val createCallback = new CreateCallback + private val ephemeralWatcher = new EphemeralWatcher + private val existsCallback = new ExistsCallback + private var stop : Boolean = false + private class CreateCallback extends StringCallback { + def processResult(rc : Int, + path : String, + ctx : Object, + name : String) { + Code.get(rc) match { + case Code.OK => { + // check that exists and wait + checkAndWatch + } + case Code.CONNECTIONLOSS => { + // TODO: Backoff + + // and try again + createAndWatch + } + case Code.NONODE => { + error("No node for path %s (could be the parent missing)".format(path)) + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + } + } + } + } + + private class EphemeralWatcher extends Watcher { + def process(event : WatchedEvent) { + // if node deleted, then recreate it + if(!stop && event.getType == Watcher.Event.EventType.NodeDeleted) + createAndWatch + } + } + + private class ExistsCallback extends StatCallback { + def processResult(rc : Int, + path : String, + ctx : Object, + stat : Stat) { + Code.get(rc) match { + case Code.OK => {} + case Code.CONNECTIONLOSS => { + // Backoff and try again + checkAndWatch + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while checking if registration node exists %s %s".format(path, Code.get(rc))) + } + } + } + } + + private def checkAndWatch() { + zkHandle.exists(path, + ephemeralWatcher, + existsCallback, + null) + } + + private def createRecursive(prefix : String, suffix : String) { + info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + if(suffix.isEmpty()) { + zkHandle.create(prefix, + data.getBytes(), + Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL, + createCallback, + null) + } else { + zkHandle.create(prefix, + new Array[Byte](0), + Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT, + new StringCallback() { + def processResult(rc : Int, + path : String, + ctx : Object, + name : String) { + + Code.get(rc) match { + case Code.OK => { + // Nothing to do + } + case Code.NODEEXISTS => { + // Nothing to do + } + case Code.CONNECTIONLOSS => { + // TODO: Backoff + + // and try again + val suffix = ctx.asInstanceOf[String] + createRecursive(path, suffix) + } + case Code.NONODE => { + error("No node for path %s (could be the parent missing)".format(path)) + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + } + } + } + }, + suffix) + // Update prefix and suffix + val index = suffix.indexOf('/', 1) match { + case -1 => suffix.length + case x : Int => x + } + // Get new prefix + val newPrefix = prefix + suffix.substring(0, index) + // Get new suffix + val newSuffix = suffix.substring(index, suffix.length) + createRecursive(newPrefix, newSuffix) + } + } + + def createAndWatch() { + val index = path.indexOf('/', 1) match { + case -1 => path.length + case x : Int => x + } + val prefix = path.substring(0, index) + val suffix = path.substring(index, path.length) + info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + createRecursive(prefix, suffix) + } + + def halt() { + stop = true + } +} From b8f901b6478d4ac9c961e899d702e6fc11cfee07 Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Sun, 23 Aug 2015 14:55:11 +0100 Subject: [PATCH 02/25] KAFKA-1387: First cut, node dependency on curator --- .../consumer/ZookeeperConsumerConnector.scala | 19 +- core/src/main/scala/kafka/utils/ZkUtils.scala | 168 +++++++++++++++++- 2 files changed, 180 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index e42d10488f8df..9646b62f03698 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -34,9 +34,10 @@ import kafka.network.BlockingChannel import kafka.serializer._ import kafka.utils.CoreUtils.inLock import kafka.utils.ZkUtils._ +import kafka.utils.ZKWatchedEphemeral import kafka.utils._ import org.I0Itec.zkclient.exception.ZkNodeExistsException -import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient} +import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection} import org.apache.zookeeper.Watcher.Event.KeeperState import scala.collection._ @@ -90,6 +91,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private val rebalanceLock = new Object private var fetcher: Option[ConsumerFetcherManager] = None private var zkClient: ZkClient = null + private var zkConnection : ZkConnection = null private var topicRegistry = new Pool[String, Pool[Int, PartitionTopicInfo]] private val checkpointedZkOffsets = new Pool[TopicAndPartition, Long] private val topicThreadIdAndQueues = new Pool[(String, ConsumerThreadId), BlockingQueue[FetchedDataChunk]] @@ -178,7 +180,9 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private def connectZk() { info("Connecting to zookeeper instance at " + config.zkConnect) - zkClient = ZkUtils.createZkClient(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + var (client, connection) = ZkUtils.createZkClientAndConnection(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + zkClient = client + zkConnection = connection } // Blocks until the offset manager is located and a channel is established to it. @@ -255,15 +259,20 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, // this API is used by unit tests only def getTopicRegistry: Pool[String, Pool[Int, PartitionTopicInfo]] = topicRegistry - + private var zkWatchedEphemeral : ZKWatchedEphemeral = null private def registerConsumerInZK(dirs: ZKGroupDirs, consumerIdString: String, topicCount: TopicCount) { info("begin registering consumer " + consumerIdString + " in ZK") val timestamp = SystemTime.milliseconds.toString val consumerRegistrationInfo = Json.encode(Map("version" -> 1, "subscription" -> topicCount.getTopicCountMap, "pattern" -> topicCount.pattern, "timestamp" -> timestamp)) - createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, - (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) + //createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, + // (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) + + if(zkWatchedEphemeral != null) + zkWatchedEphemeral.halt + zkWatchedEphemeral = new ZKWatchedEphemeral(dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, zkConnection) + zkWatchedEphemeral.createAndWatch info("end registering consumer " + consumerIdString + " in ZK") } diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 74b587e04cdd6..2aa900631aa9b 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -20,7 +20,7 @@ package kafka.utils import kafka.cluster._ import kafka.consumer.{ConsumerThreadId, TopicCount} import kafka.server.ConfigType -import org.I0Itec.zkclient.ZkClient +import org.I0Itec.zkclient.{ZkClient,ZkConnection} import org.I0Itec.zkclient.exception.{ZkNodeExistsException, ZkNoNodeException, ZkMarshallingError, ZkBadVersionException} import org.I0Itec.zkclient.serialize.ZkSerializer @@ -36,6 +36,15 @@ import kafka.controller.KafkaController import kafka.controller.LeaderIsrAndControllerEpoch import kafka.common.TopicAndPartition +import org.apache.zookeeper.AsyncCallback.StatCallback +import org.apache.zookeeper.AsyncCallback.StringCallback +import org.apache.zookeeper.CreateMode +import org.apache.zookeeper.KeeperException.Code +import org.apache.zookeeper.WatchedEvent +import org.apache.zookeeper.Watcher +import org.apache.zookeeper.ZooDefs.Ids +import org.apache.zookeeper.ZooKeeper + object ZkUtils extends Logging { val ConsumersPath = "/consumers" val BrokerIdsPath = "/brokers/ids" @@ -809,7 +818,13 @@ object ZkUtils extends Logging { def createZkClient(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): ZkClient = { val zkClient = new ZkClient(zkUrl, sessionTimeout, connectionTimeout, ZKStringSerializer) - zkClient + zkClient + } + + def createZkClientAndConnection(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): (ZkClient, ZkConnection) = { + val zkConnection = new ZkConnection(zkUrl, sessionTimeout) + val zkClient = new ZkClient(zkConnection, connectionTimeout, ZKStringSerializer) + (zkClient, zkConnection) } } @@ -892,3 +907,152 @@ object ZkPath { client.createPersistentSequential(path, data) } } + +class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConnection : ZkConnection) extends Logging { + private val path = ephemeralPath + private val data = ephemeralData + private val zkHandle = zkConnection.getZookeeper + private val createCallback = new CreateCallback + private val ephemeralWatcher = new EphemeralWatcher + private val existsCallback = new ExistsCallback + private var stop : Boolean = false + private class CreateCallback extends StringCallback { + def processResult(rc : Int, + path : String, + ctx : Object, + name : String) { + Code.get(rc) match { + case Code.OK => { + // check that exists and wait + checkAndWatch + } + case Code.CONNECTIONLOSS => { + // TODO: Backoff + + // and try again + createAndWatch + } + case Code.NONODE => { + error("No node for path %s (could be the parent missing)".format(path)) + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + } + } + } + } + + private class EphemeralWatcher extends Watcher { + def process(event : WatchedEvent) { + // if node deleted, then recreate it + if(!stop && event.getType == Watcher.Event.EventType.NodeDeleted) + createAndWatch + } + } + + private class ExistsCallback extends StatCallback { + def processResult(rc : Int, + path : String, + ctx : Object, + stat : Stat) { + Code.get(rc) match { + case Code.OK => {} + case Code.CONNECTIONLOSS => { + // Backoff and try again + checkAndWatch + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while checking if registration node exists %s %s".format(path, Code.get(rc))) + } + } + } + } + + private def checkAndWatch() { + zkHandle.exists(path, + ephemeralWatcher, + existsCallback, + null) + } + + private def createRecursive(prefix : String, suffix : String) { + info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + if(suffix.isEmpty()) { + zkHandle.create(prefix, + data.getBytes(), + Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL, + createCallback, + null) + } else { + zkHandle.create(prefix, + new Array[Byte](0), + Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT, + new StringCallback() { + def processResult(rc : Int, + path : String, + ctx : Object, + name : String) { + + Code.get(rc) match { + case Code.OK => { + // Nothing to do + } + case Code.NODEEXISTS => { + // Nothing to do + } + case Code.CONNECTIONLOSS => { + // TODO: Backoff + + // and try again + val suffix = ctx.asInstanceOf[String] + createRecursive(path, suffix) + } + case Code.NONODE => { + error("No node for path %s (could be the parent missing)".format(path)) + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + } + } + } + }, + suffix) + // Update prefix and suffix + val index = suffix.indexOf('/', 1) match { + case -1 => suffix.length + case x : Int => x + } + // Get new prefix + val newPrefix = prefix + suffix.substring(0, index) + // Get new suffix + val newSuffix = suffix.substring(index, suffix.length) + createRecursive(newPrefix, newSuffix) + } + } + + def createAndWatch() { + val index = path.indexOf('/', 1) match { + case -1 => path.length + case x : Int => x + } + val prefix = path.substring(0, index) + val suffix = path.substring(index, path.length) + info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + createRecursive(prefix, suffix) + } + + def halt() { + stop = true + } +} From f03c301d5d919d9c05c6837de508b4f383906fdb Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Sun, 23 Aug 2015 14:55:11 +0100 Subject: [PATCH 03/25] KAFKA-1387: First cut, node dependency on curator --- .../consumer/ZookeeperConsumerConnector.scala | 19 +- core/src/main/scala/kafka/utils/ZkUtils.scala | 168 +++++++++++++++++- 2 files changed, 180 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index e42d10488f8df..9646b62f03698 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -34,9 +34,10 @@ import kafka.network.BlockingChannel import kafka.serializer._ import kafka.utils.CoreUtils.inLock import kafka.utils.ZkUtils._ +import kafka.utils.ZKWatchedEphemeral import kafka.utils._ import org.I0Itec.zkclient.exception.ZkNodeExistsException -import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient} +import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection} import org.apache.zookeeper.Watcher.Event.KeeperState import scala.collection._ @@ -90,6 +91,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private val rebalanceLock = new Object private var fetcher: Option[ConsumerFetcherManager] = None private var zkClient: ZkClient = null + private var zkConnection : ZkConnection = null private var topicRegistry = new Pool[String, Pool[Int, PartitionTopicInfo]] private val checkpointedZkOffsets = new Pool[TopicAndPartition, Long] private val topicThreadIdAndQueues = new Pool[(String, ConsumerThreadId), BlockingQueue[FetchedDataChunk]] @@ -178,7 +180,9 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private def connectZk() { info("Connecting to zookeeper instance at " + config.zkConnect) - zkClient = ZkUtils.createZkClient(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + var (client, connection) = ZkUtils.createZkClientAndConnection(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + zkClient = client + zkConnection = connection } // Blocks until the offset manager is located and a channel is established to it. @@ -255,15 +259,20 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, // this API is used by unit tests only def getTopicRegistry: Pool[String, Pool[Int, PartitionTopicInfo]] = topicRegistry - + private var zkWatchedEphemeral : ZKWatchedEphemeral = null private def registerConsumerInZK(dirs: ZKGroupDirs, consumerIdString: String, topicCount: TopicCount) { info("begin registering consumer " + consumerIdString + " in ZK") val timestamp = SystemTime.milliseconds.toString val consumerRegistrationInfo = Json.encode(Map("version" -> 1, "subscription" -> topicCount.getTopicCountMap, "pattern" -> topicCount.pattern, "timestamp" -> timestamp)) - createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, - (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) + //createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, + // (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) + + if(zkWatchedEphemeral != null) + zkWatchedEphemeral.halt + zkWatchedEphemeral = new ZKWatchedEphemeral(dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, zkConnection) + zkWatchedEphemeral.createAndWatch info("end registering consumer " + consumerIdString + " in ZK") } diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 74b587e04cdd6..2aa900631aa9b 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -20,7 +20,7 @@ package kafka.utils import kafka.cluster._ import kafka.consumer.{ConsumerThreadId, TopicCount} import kafka.server.ConfigType -import org.I0Itec.zkclient.ZkClient +import org.I0Itec.zkclient.{ZkClient,ZkConnection} import org.I0Itec.zkclient.exception.{ZkNodeExistsException, ZkNoNodeException, ZkMarshallingError, ZkBadVersionException} import org.I0Itec.zkclient.serialize.ZkSerializer @@ -36,6 +36,15 @@ import kafka.controller.KafkaController import kafka.controller.LeaderIsrAndControllerEpoch import kafka.common.TopicAndPartition +import org.apache.zookeeper.AsyncCallback.StatCallback +import org.apache.zookeeper.AsyncCallback.StringCallback +import org.apache.zookeeper.CreateMode +import org.apache.zookeeper.KeeperException.Code +import org.apache.zookeeper.WatchedEvent +import org.apache.zookeeper.Watcher +import org.apache.zookeeper.ZooDefs.Ids +import org.apache.zookeeper.ZooKeeper + object ZkUtils extends Logging { val ConsumersPath = "/consumers" val BrokerIdsPath = "/brokers/ids" @@ -809,7 +818,13 @@ object ZkUtils extends Logging { def createZkClient(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): ZkClient = { val zkClient = new ZkClient(zkUrl, sessionTimeout, connectionTimeout, ZKStringSerializer) - zkClient + zkClient + } + + def createZkClientAndConnection(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): (ZkClient, ZkConnection) = { + val zkConnection = new ZkConnection(zkUrl, sessionTimeout) + val zkClient = new ZkClient(zkConnection, connectionTimeout, ZKStringSerializer) + (zkClient, zkConnection) } } @@ -892,3 +907,152 @@ object ZkPath { client.createPersistentSequential(path, data) } } + +class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConnection : ZkConnection) extends Logging { + private val path = ephemeralPath + private val data = ephemeralData + private val zkHandle = zkConnection.getZookeeper + private val createCallback = new CreateCallback + private val ephemeralWatcher = new EphemeralWatcher + private val existsCallback = new ExistsCallback + private var stop : Boolean = false + private class CreateCallback extends StringCallback { + def processResult(rc : Int, + path : String, + ctx : Object, + name : String) { + Code.get(rc) match { + case Code.OK => { + // check that exists and wait + checkAndWatch + } + case Code.CONNECTIONLOSS => { + // TODO: Backoff + + // and try again + createAndWatch + } + case Code.NONODE => { + error("No node for path %s (could be the parent missing)".format(path)) + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + } + } + } + } + + private class EphemeralWatcher extends Watcher { + def process(event : WatchedEvent) { + // if node deleted, then recreate it + if(!stop && event.getType == Watcher.Event.EventType.NodeDeleted) + createAndWatch + } + } + + private class ExistsCallback extends StatCallback { + def processResult(rc : Int, + path : String, + ctx : Object, + stat : Stat) { + Code.get(rc) match { + case Code.OK => {} + case Code.CONNECTIONLOSS => { + // Backoff and try again + checkAndWatch + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while checking if registration node exists %s %s".format(path, Code.get(rc))) + } + } + } + } + + private def checkAndWatch() { + zkHandle.exists(path, + ephemeralWatcher, + existsCallback, + null) + } + + private def createRecursive(prefix : String, suffix : String) { + info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + if(suffix.isEmpty()) { + zkHandle.create(prefix, + data.getBytes(), + Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL, + createCallback, + null) + } else { + zkHandle.create(prefix, + new Array[Byte](0), + Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT, + new StringCallback() { + def processResult(rc : Int, + path : String, + ctx : Object, + name : String) { + + Code.get(rc) match { + case Code.OK => { + // Nothing to do + } + case Code.NODEEXISTS => { + // Nothing to do + } + case Code.CONNECTIONLOSS => { + // TODO: Backoff + + // and try again + val suffix = ctx.asInstanceOf[String] + createRecursive(path, suffix) + } + case Code.NONODE => { + error("No node for path %s (could be the parent missing)".format(path)) + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + } + } + } + }, + suffix) + // Update prefix and suffix + val index = suffix.indexOf('/', 1) match { + case -1 => suffix.length + case x : Int => x + } + // Get new prefix + val newPrefix = prefix + suffix.substring(0, index) + // Get new suffix + val newSuffix = suffix.substring(index, suffix.length) + createRecursive(newPrefix, newSuffix) + } + } + + def createAndWatch() { + val index = path.indexOf('/', 1) match { + case -1 => path.length + case x : Int => x + } + val prefix = path.substring(0, index) + val suffix = path.substring(index, path.length) + info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + createRecursive(prefix, suffix) + } + + def halt() { + stop = true + } +} From d8eab9e0f569eaaecb4afda4d486d00600ad1e6f Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Mon, 24 Aug 2015 15:56:01 +0100 Subject: [PATCH 04/25] KAFKA-1387: Some polishing --- .../consumer/ZookeeperConsumerConnector.scala | 4 +++- core/src/main/scala/kafka/utils/ZkUtils.scala | 20 ++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index 9646b62f03698..d1cd03c07ef03 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -271,7 +271,9 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, if(zkWatchedEphemeral != null) zkWatchedEphemeral.halt - zkWatchedEphemeral = new ZKWatchedEphemeral(dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, zkConnection) + zkWatchedEphemeral = new ZKWatchedEphemeral(dirs. + consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, + zkConnection.getZookeeper) zkWatchedEphemeral.createAndWatch info("end registering consumer " + consumerIdString + " in ZK") } diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 2aa900631aa9b..28780bd4290b1 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -908,13 +908,13 @@ object ZkPath { } } -class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConnection : ZkConnection) extends Logging { - private val path = ephemeralPath - private val data = ephemeralData - private val zkHandle = zkConnection.getZookeeper +class ZKWatchedEphemeral(path : String, + data : String, + zkHandle : ZooKeeper) extends Logging { private val createCallback = new CreateCallback private val ephemeralWatcher = new EphemeralWatcher private val existsCallback = new ExistsCallback + @volatile private var stop : Boolean = false private class CreateCallback extends StringCallback { def processResult(rc : Int, @@ -927,9 +927,7 @@ class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConne checkAndWatch } case Code.CONNECTIONLOSS => { - // TODO: Backoff - - // and try again + // try again createAndWatch } case Code.NONODE => { @@ -982,7 +980,7 @@ class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConne } private def createRecursive(prefix : String, suffix : String) { - info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) if(suffix.isEmpty()) { zkHandle.create(prefix, data.getBytes(), @@ -1009,9 +1007,7 @@ class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConne // Nothing to do } case Code.CONNECTIONLOSS => { - // TODO: Backoff - - // and try again + // try again val suffix = ctx.asInstanceOf[String] createRecursive(path, suffix) } @@ -1048,7 +1044,7 @@ class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConne } val prefix = path.substring(0, index) val suffix = path.substring(index, path.length) - info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) createRecursive(prefix, suffix) } From b7cbe5dbecbc28a564b99209114f39db785c73dd Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Mon, 24 Aug 2015 16:50:58 +0100 Subject: [PATCH 05/25] KAFKA-1387: Style fixes --- .../consumer/ZookeeperConsumerConnector.scala | 4 +- core/src/main/scala/kafka/utils/ZkUtils.scala | 53 +++++++++---------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index d1cd03c07ef03..5e30e6ebd3dc8 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -268,11 +268,11 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, //createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, // (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) - + if(zkWatchedEphemeral != null) zkWatchedEphemeral.halt zkWatchedEphemeral = new ZKWatchedEphemeral(dirs. - consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, + consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, zkConnection.getZookeeper) zkWatchedEphemeral.createAndWatch info("end registering consumer " + consumerIdString + " in ZK") diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 28780bd4290b1..5227ad85b4195 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -824,7 +824,7 @@ object ZkUtils extends Logging { def createZkClientAndConnection(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): (ZkClient, ZkConnection) = { val zkConnection = new ZkConnection(zkUrl, sessionTimeout) val zkClient = new ZkClient(zkConnection, connectionTimeout, ZKStringSerializer) - (zkClient, zkConnection) + (zkClient, zkConnection) } } @@ -908,13 +908,13 @@ object ZkPath { } } -class ZKWatchedEphemeral(path : String, - data : String, +class ZKWatchedEphemeral(path : String, + data : String, zkHandle : ZooKeeper) extends Logging { private val createCallback = new CreateCallback private val ephemeralWatcher = new EphemeralWatcher private val existsCallback = new ExistsCallback - @volatile + @volatile private var stop : Boolean = false private class CreateCallback extends StringCallback { def processResult(rc : Int, @@ -934,7 +934,7 @@ class ZKWatchedEphemeral(path : String, error("No node for path %s (could be the parent missing)".format(path)) } case Code.SESSIONEXPIRED => { - error("Session has expired while creating %s".format(path)) + error("Session has expired while creating %s".format(path)) } case _ => { info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) @@ -942,7 +942,7 @@ class ZKWatchedEphemeral(path : String, } } } - + private class EphemeralWatcher extends Watcher { def process(event : WatchedEvent) { // if node deleted, then recreate it @@ -950,7 +950,7 @@ class ZKWatchedEphemeral(path : String, createAndWatch } } - + private class ExistsCallback extends StatCallback { def processResult(rc : Int, path : String, @@ -963,7 +963,7 @@ class ZKWatchedEphemeral(path : String, checkAndWatch } case Code.SESSIONEXPIRED => { - error("Session has expired while creating %s".format(path)) + error("Session has expired while creating %s".format(path)) } case _ => { info("ZooKeeper event while checking if registration node exists %s %s".format(path, Code.get(rc))) @@ -971,34 +971,33 @@ class ZKWatchedEphemeral(path : String, } } } - + private def checkAndWatch() { - zkHandle.exists(path, - ephemeralWatcher, + zkHandle.exists(path, + ephemeralWatcher, existsCallback, - null) + null) } - + private def createRecursive(prefix : String, suffix : String) { debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) if(suffix.isEmpty()) { - zkHandle.create(prefix, - data.getBytes(), - Ids.OPEN_ACL_UNSAFE, - CreateMode.EPHEMERAL, + zkHandle.create(prefix, + data.getBytes(), + Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL, createCallback, null) } else { - zkHandle.create(prefix, - new Array[Byte](0), - Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT, + zkHandle.create(prefix, + new Array[Byte](0), + Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT, new StringCallback() { def processResult(rc : Int, path : String, ctx : Object, name : String) { - Code.get(rc) match { case Code.OK => { // Nothing to do @@ -1015,7 +1014,7 @@ class ZKWatchedEphemeral(path : String, error("No node for path %s (could be the parent missing)".format(path)) } case Code.SESSIONEXPIRED => { - error("Session has expired while creating %s".format(path)) + error("Session has expired while creating %s".format(path)) } case _ => { info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) @@ -1034,9 +1033,9 @@ class ZKWatchedEphemeral(path : String, // Get new suffix val newSuffix = suffix.substring(index, suffix.length) createRecursive(newPrefix, newSuffix) - } + } } - + def createAndWatch() { val index = path.indexOf('/', 1) match { case -1 => path.length @@ -1045,9 +1044,9 @@ class ZKWatchedEphemeral(path : String, val prefix = path.substring(0, index) val suffix = path.substring(index, path.length) debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) - createRecursive(prefix, suffix) + createRecursive(prefix, suffix) } - + def halt() { stop = true } From 336f67c641c44b73ac1dbb66cdde4ff97f2fcd9a Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Mon, 24 Aug 2015 16:53:18 +0100 Subject: [PATCH 06/25] KAFKA-1387: More style fixes --- core/src/main/scala/kafka/utils/ZkUtils.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 5227ad85b4195..2ede77f84db17 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -820,7 +820,7 @@ object ZkUtils extends Logging { val zkClient = new ZkClient(zkUrl, sessionTimeout, connectionTimeout, ZKStringSerializer) zkClient } - + def createZkClientAndConnection(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): (ZkClient, ZkConnection) = { val zkConnection = new ZkConnection(zkUrl, sessionTimeout) val zkClient = new ZkClient(zkConnection, connectionTimeout, ZKStringSerializer) @@ -940,9 +940,9 @@ class ZKWatchedEphemeral(path : String, info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) } } - } + } } - + private class EphemeralWatcher extends Watcher { def process(event : WatchedEvent) { // if node deleted, then recreate it From 9961665230e04331f7767d8aa8aaac0a14f46cd8 Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Sun, 23 Aug 2015 14:55:11 +0100 Subject: [PATCH 07/25] KAFKA-1387: First cut, node dependency on curator --- .../consumer/ZookeeperConsumerConnector.scala | 19 +- core/src/main/scala/kafka/utils/ZkUtils.scala | 168 +++++++++++++++++- 2 files changed, 180 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index e42d10488f8df..9646b62f03698 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -34,9 +34,10 @@ import kafka.network.BlockingChannel import kafka.serializer._ import kafka.utils.CoreUtils.inLock import kafka.utils.ZkUtils._ +import kafka.utils.ZKWatchedEphemeral import kafka.utils._ import org.I0Itec.zkclient.exception.ZkNodeExistsException -import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient} +import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection} import org.apache.zookeeper.Watcher.Event.KeeperState import scala.collection._ @@ -90,6 +91,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private val rebalanceLock = new Object private var fetcher: Option[ConsumerFetcherManager] = None private var zkClient: ZkClient = null + private var zkConnection : ZkConnection = null private var topicRegistry = new Pool[String, Pool[Int, PartitionTopicInfo]] private val checkpointedZkOffsets = new Pool[TopicAndPartition, Long] private val topicThreadIdAndQueues = new Pool[(String, ConsumerThreadId), BlockingQueue[FetchedDataChunk]] @@ -178,7 +180,9 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private def connectZk() { info("Connecting to zookeeper instance at " + config.zkConnect) - zkClient = ZkUtils.createZkClient(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + var (client, connection) = ZkUtils.createZkClientAndConnection(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + zkClient = client + zkConnection = connection } // Blocks until the offset manager is located and a channel is established to it. @@ -255,15 +259,20 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, // this API is used by unit tests only def getTopicRegistry: Pool[String, Pool[Int, PartitionTopicInfo]] = topicRegistry - + private var zkWatchedEphemeral : ZKWatchedEphemeral = null private def registerConsumerInZK(dirs: ZKGroupDirs, consumerIdString: String, topicCount: TopicCount) { info("begin registering consumer " + consumerIdString + " in ZK") val timestamp = SystemTime.milliseconds.toString val consumerRegistrationInfo = Json.encode(Map("version" -> 1, "subscription" -> topicCount.getTopicCountMap, "pattern" -> topicCount.pattern, "timestamp" -> timestamp)) - createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, - (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) + //createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, + // (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) + + if(zkWatchedEphemeral != null) + zkWatchedEphemeral.halt + zkWatchedEphemeral = new ZKWatchedEphemeral(dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, zkConnection) + zkWatchedEphemeral.createAndWatch info("end registering consumer " + consumerIdString + " in ZK") } diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 74b587e04cdd6..2aa900631aa9b 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -20,7 +20,7 @@ package kafka.utils import kafka.cluster._ import kafka.consumer.{ConsumerThreadId, TopicCount} import kafka.server.ConfigType -import org.I0Itec.zkclient.ZkClient +import org.I0Itec.zkclient.{ZkClient,ZkConnection} import org.I0Itec.zkclient.exception.{ZkNodeExistsException, ZkNoNodeException, ZkMarshallingError, ZkBadVersionException} import org.I0Itec.zkclient.serialize.ZkSerializer @@ -36,6 +36,15 @@ import kafka.controller.KafkaController import kafka.controller.LeaderIsrAndControllerEpoch import kafka.common.TopicAndPartition +import org.apache.zookeeper.AsyncCallback.StatCallback +import org.apache.zookeeper.AsyncCallback.StringCallback +import org.apache.zookeeper.CreateMode +import org.apache.zookeeper.KeeperException.Code +import org.apache.zookeeper.WatchedEvent +import org.apache.zookeeper.Watcher +import org.apache.zookeeper.ZooDefs.Ids +import org.apache.zookeeper.ZooKeeper + object ZkUtils extends Logging { val ConsumersPath = "/consumers" val BrokerIdsPath = "/brokers/ids" @@ -809,7 +818,13 @@ object ZkUtils extends Logging { def createZkClient(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): ZkClient = { val zkClient = new ZkClient(zkUrl, sessionTimeout, connectionTimeout, ZKStringSerializer) - zkClient + zkClient + } + + def createZkClientAndConnection(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): (ZkClient, ZkConnection) = { + val zkConnection = new ZkConnection(zkUrl, sessionTimeout) + val zkClient = new ZkClient(zkConnection, connectionTimeout, ZKStringSerializer) + (zkClient, zkConnection) } } @@ -892,3 +907,152 @@ object ZkPath { client.createPersistentSequential(path, data) } } + +class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConnection : ZkConnection) extends Logging { + private val path = ephemeralPath + private val data = ephemeralData + private val zkHandle = zkConnection.getZookeeper + private val createCallback = new CreateCallback + private val ephemeralWatcher = new EphemeralWatcher + private val existsCallback = new ExistsCallback + private var stop : Boolean = false + private class CreateCallback extends StringCallback { + def processResult(rc : Int, + path : String, + ctx : Object, + name : String) { + Code.get(rc) match { + case Code.OK => { + // check that exists and wait + checkAndWatch + } + case Code.CONNECTIONLOSS => { + // TODO: Backoff + + // and try again + createAndWatch + } + case Code.NONODE => { + error("No node for path %s (could be the parent missing)".format(path)) + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + } + } + } + } + + private class EphemeralWatcher extends Watcher { + def process(event : WatchedEvent) { + // if node deleted, then recreate it + if(!stop && event.getType == Watcher.Event.EventType.NodeDeleted) + createAndWatch + } + } + + private class ExistsCallback extends StatCallback { + def processResult(rc : Int, + path : String, + ctx : Object, + stat : Stat) { + Code.get(rc) match { + case Code.OK => {} + case Code.CONNECTIONLOSS => { + // Backoff and try again + checkAndWatch + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while checking if registration node exists %s %s".format(path, Code.get(rc))) + } + } + } + } + + private def checkAndWatch() { + zkHandle.exists(path, + ephemeralWatcher, + existsCallback, + null) + } + + private def createRecursive(prefix : String, suffix : String) { + info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + if(suffix.isEmpty()) { + zkHandle.create(prefix, + data.getBytes(), + Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL, + createCallback, + null) + } else { + zkHandle.create(prefix, + new Array[Byte](0), + Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT, + new StringCallback() { + def processResult(rc : Int, + path : String, + ctx : Object, + name : String) { + + Code.get(rc) match { + case Code.OK => { + // Nothing to do + } + case Code.NODEEXISTS => { + // Nothing to do + } + case Code.CONNECTIONLOSS => { + // TODO: Backoff + + // and try again + val suffix = ctx.asInstanceOf[String] + createRecursive(path, suffix) + } + case Code.NONODE => { + error("No node for path %s (could be the parent missing)".format(path)) + } + case Code.SESSIONEXPIRED => { + error("Session has expired while creating %s".format(path)) + } + case _ => { + info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + } + } + } + }, + suffix) + // Update prefix and suffix + val index = suffix.indexOf('/', 1) match { + case -1 => suffix.length + case x : Int => x + } + // Get new prefix + val newPrefix = prefix + suffix.substring(0, index) + // Get new suffix + val newSuffix = suffix.substring(index, suffix.length) + createRecursive(newPrefix, newSuffix) + } + } + + def createAndWatch() { + val index = path.indexOf('/', 1) match { + case -1 => path.length + case x : Int => x + } + val prefix = path.substring(0, index) + val suffix = path.substring(index, path.length) + info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + createRecursive(prefix, suffix) + } + + def halt() { + stop = true + } +} From b52c12422f7a831137d8659f14779eaad1972217 Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Mon, 24 Aug 2015 15:56:01 +0100 Subject: [PATCH 08/25] KAFKA-1387: Some polishing --- .../consumer/ZookeeperConsumerConnector.scala | 4 +++- core/src/main/scala/kafka/utils/ZkUtils.scala | 20 ++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index 9646b62f03698..d1cd03c07ef03 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -271,7 +271,9 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, if(zkWatchedEphemeral != null) zkWatchedEphemeral.halt - zkWatchedEphemeral = new ZKWatchedEphemeral(dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, zkConnection) + zkWatchedEphemeral = new ZKWatchedEphemeral(dirs. + consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, + zkConnection.getZookeeper) zkWatchedEphemeral.createAndWatch info("end registering consumer " + consumerIdString + " in ZK") } diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 2aa900631aa9b..28780bd4290b1 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -908,13 +908,13 @@ object ZkPath { } } -class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConnection : ZkConnection) extends Logging { - private val path = ephemeralPath - private val data = ephemeralData - private val zkHandle = zkConnection.getZookeeper +class ZKWatchedEphemeral(path : String, + data : String, + zkHandle : ZooKeeper) extends Logging { private val createCallback = new CreateCallback private val ephemeralWatcher = new EphemeralWatcher private val existsCallback = new ExistsCallback + @volatile private var stop : Boolean = false private class CreateCallback extends StringCallback { def processResult(rc : Int, @@ -927,9 +927,7 @@ class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConne checkAndWatch } case Code.CONNECTIONLOSS => { - // TODO: Backoff - - // and try again + // try again createAndWatch } case Code.NONODE => { @@ -982,7 +980,7 @@ class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConne } private def createRecursive(prefix : String, suffix : String) { - info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) if(suffix.isEmpty()) { zkHandle.create(prefix, data.getBytes(), @@ -1009,9 +1007,7 @@ class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConne // Nothing to do } case Code.CONNECTIONLOSS => { - // TODO: Backoff - - // and try again + // try again val suffix = ctx.asInstanceOf[String] createRecursive(path, suffix) } @@ -1048,7 +1044,7 @@ class ZKWatchedEphemeral(ephemeralPath : String, ephemeralData : String, zkConne } val prefix = path.substring(0, index) val suffix = path.substring(index, path.length) - info("### Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) createRecursive(prefix, suffix) } From b2400a0a37555250d50b1f1abfdda2c4d00b03ac Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Mon, 24 Aug 2015 16:50:58 +0100 Subject: [PATCH 09/25] KAFKA-1387: Style fixes --- .../consumer/ZookeeperConsumerConnector.scala | 4 +- core/src/main/scala/kafka/utils/ZkUtils.scala | 53 +++++++++---------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index d1cd03c07ef03..5e30e6ebd3dc8 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -268,11 +268,11 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, //createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, // (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) - + if(zkWatchedEphemeral != null) zkWatchedEphemeral.halt zkWatchedEphemeral = new ZKWatchedEphemeral(dirs. - consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, + consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, zkConnection.getZookeeper) zkWatchedEphemeral.createAndWatch info("end registering consumer " + consumerIdString + " in ZK") diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 28780bd4290b1..5227ad85b4195 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -824,7 +824,7 @@ object ZkUtils extends Logging { def createZkClientAndConnection(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): (ZkClient, ZkConnection) = { val zkConnection = new ZkConnection(zkUrl, sessionTimeout) val zkClient = new ZkClient(zkConnection, connectionTimeout, ZKStringSerializer) - (zkClient, zkConnection) + (zkClient, zkConnection) } } @@ -908,13 +908,13 @@ object ZkPath { } } -class ZKWatchedEphemeral(path : String, - data : String, +class ZKWatchedEphemeral(path : String, + data : String, zkHandle : ZooKeeper) extends Logging { private val createCallback = new CreateCallback private val ephemeralWatcher = new EphemeralWatcher private val existsCallback = new ExistsCallback - @volatile + @volatile private var stop : Boolean = false private class CreateCallback extends StringCallback { def processResult(rc : Int, @@ -934,7 +934,7 @@ class ZKWatchedEphemeral(path : String, error("No node for path %s (could be the parent missing)".format(path)) } case Code.SESSIONEXPIRED => { - error("Session has expired while creating %s".format(path)) + error("Session has expired while creating %s".format(path)) } case _ => { info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) @@ -942,7 +942,7 @@ class ZKWatchedEphemeral(path : String, } } } - + private class EphemeralWatcher extends Watcher { def process(event : WatchedEvent) { // if node deleted, then recreate it @@ -950,7 +950,7 @@ class ZKWatchedEphemeral(path : String, createAndWatch } } - + private class ExistsCallback extends StatCallback { def processResult(rc : Int, path : String, @@ -963,7 +963,7 @@ class ZKWatchedEphemeral(path : String, checkAndWatch } case Code.SESSIONEXPIRED => { - error("Session has expired while creating %s".format(path)) + error("Session has expired while creating %s".format(path)) } case _ => { info("ZooKeeper event while checking if registration node exists %s %s".format(path, Code.get(rc))) @@ -971,34 +971,33 @@ class ZKWatchedEphemeral(path : String, } } } - + private def checkAndWatch() { - zkHandle.exists(path, - ephemeralWatcher, + zkHandle.exists(path, + ephemeralWatcher, existsCallback, - null) + null) } - + private def createRecursive(prefix : String, suffix : String) { debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) if(suffix.isEmpty()) { - zkHandle.create(prefix, - data.getBytes(), - Ids.OPEN_ACL_UNSAFE, - CreateMode.EPHEMERAL, + zkHandle.create(prefix, + data.getBytes(), + Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL, createCallback, null) } else { - zkHandle.create(prefix, - new Array[Byte](0), - Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT, + zkHandle.create(prefix, + new Array[Byte](0), + Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT, new StringCallback() { def processResult(rc : Int, path : String, ctx : Object, name : String) { - Code.get(rc) match { case Code.OK => { // Nothing to do @@ -1015,7 +1014,7 @@ class ZKWatchedEphemeral(path : String, error("No node for path %s (could be the parent missing)".format(path)) } case Code.SESSIONEXPIRED => { - error("Session has expired while creating %s".format(path)) + error("Session has expired while creating %s".format(path)) } case _ => { info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) @@ -1034,9 +1033,9 @@ class ZKWatchedEphemeral(path : String, // Get new suffix val newSuffix = suffix.substring(index, suffix.length) createRecursive(newPrefix, newSuffix) - } + } } - + def createAndWatch() { val index = path.indexOf('/', 1) match { case -1 => path.length @@ -1045,9 +1044,9 @@ class ZKWatchedEphemeral(path : String, val prefix = path.substring(0, index) val suffix = path.substring(index, path.length) debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) - createRecursive(prefix, suffix) + createRecursive(prefix, suffix) } - + def halt() { stop = true } From 888f6e0cf17d6a3a8d6b8dd46f8099731ba36511 Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Mon, 24 Aug 2015 16:53:18 +0100 Subject: [PATCH 10/25] KAFKA-1387: More style fixes --- core/src/main/scala/kafka/utils/ZkUtils.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 5227ad85b4195..2ede77f84db17 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -820,7 +820,7 @@ object ZkUtils extends Logging { val zkClient = new ZkClient(zkUrl, sessionTimeout, connectionTimeout, ZKStringSerializer) zkClient } - + def createZkClientAndConnection(zkUrl: String, sessionTimeout: Int, connectionTimeout: Int): (ZkClient, ZkConnection) = { val zkConnection = new ZkConnection(zkUrl, sessionTimeout) val zkClient = new ZkClient(zkConnection, connectionTimeout, ZKStringSerializer) @@ -940,9 +940,9 @@ class ZKWatchedEphemeral(path : String, info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) } } - } + } } - + private class EphemeralWatcher extends Watcher { def process(event : WatchedEvent) { // if node deleted, then recreate it From d675b024b0e8627c4c2c9c113c07527851e81f7a Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Sat, 29 Aug 2015 16:00:07 +0100 Subject: [PATCH 11/25] KAFKA-1387 --- .../consumer/ZookeeperConsumerConnector.scala | 7 +++++-- .../kafka/controller/KafkaController.scala | 7 ++++--- .../scala/kafka/server/KafkaHealthcheck.scala | 16 ++++++++++++--- .../main/scala/kafka/server/KafkaServer.scala | 17 +++++++++------- .../kafka/server/ZookeeperLeaderElector.scala | 20 ++++++++++++++----- core/src/main/scala/kafka/utils/ZkUtils.scala | 18 ++++++++++------- .../scala/unit/kafka/admin/AdminTest.scala | 6 +++--- .../unit/kafka/admin/TopicCommandTest.scala | 4 ++-- .../kafka/server/LeaderElectionTest.scala | 2 +- .../scala/unit/kafka/utils/TestUtils.scala | 6 +++--- .../unit/kafka/zk/ZooKeeperTestHarness.scala | 7 +++++-- 11 files changed, 72 insertions(+), 38 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index 5e30e6ebd3dc8..8f9ad5b5ecd0a 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -216,6 +216,10 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, if (config.autoCommitEnable) commitOffsets(true) if (zkClient != null) { + if(zkWatchedEphemeral != null) { + zkWatchedEphemeral.halt + zkWatchedEphemeral = null + } zkClient.close() zkClient = null } @@ -269,8 +273,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, //createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, // (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) - if(zkWatchedEphemeral != null) - zkWatchedEphemeral.halt + assert(zkWatchedEphemeral == null) zkWatchedEphemeral = new ZKWatchedEphemeral(dirs. consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, zkConnection.getZookeeper) diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala index 4c376168c1af4..d06afbcf62882 100755 --- a/core/src/main/scala/kafka/controller/KafkaController.scala +++ b/core/src/main/scala/kafka/controller/KafkaController.scala @@ -32,7 +32,7 @@ import kafka.utils.ZkUtils._ import kafka.utils._ import kafka.utils.CoreUtils._ import org.apache.zookeeper.Watcher.Event.KeeperState -import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient} +import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection} import org.I0Itec.zkclient.exception.{ZkNodeExistsException, ZkNoNodeException} import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.locks.ReentrantLock @@ -40,6 +40,7 @@ import kafka.server._ import kafka.common.TopicAndPartition class ControllerContext(val zkClient: ZkClient, + val zkConnection : ZkConnection, val zkSessionTimeout: Int) { var controllerChannelManager: ControllerChannelManager = null val controllerLock: ReentrantLock = new ReentrantLock() @@ -149,11 +150,11 @@ object KafkaController extends Logging { } } -class KafkaController(val config : KafkaConfig, zkClient: ZkClient, val brokerState: BrokerState) extends Logging with KafkaMetricsGroup { +class KafkaController(val config : KafkaConfig, zkClient: ZkClient, zkConnection : ZkConnection, val brokerState: BrokerState) extends Logging with KafkaMetricsGroup { this.logIdent = "[Controller " + config.brokerId + "]: " private var isRunning = true private val stateChangeLogger = KafkaController.stateChangeLogger - val controllerContext = new ControllerContext(zkClient, config.zkSessionTimeoutMs) + val controllerContext = new ControllerContext(zkClient, zkConnection, config.zkSessionTimeoutMs) val partitionStateMachine = new PartitionStateMachine(this) val replicaStateMachine = new ReplicaStateMachine(this) private val controllerElector = new ZookeeperLeaderElector(controllerContext, ZkUtils.ControllerPath, onControllerFailover, diff --git a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala index e6e270bbdea6c..5f3e722308db5 100644 --- a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala +++ b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala @@ -21,7 +21,7 @@ import kafka.cluster.EndPoint import kafka.utils._ import org.apache.kafka.common.protocol.SecurityProtocol import org.apache.zookeeper.Watcher.Event.KeeperState -import org.I0Itec.zkclient.{IZkStateListener, ZkClient} +import org.I0Itec.zkclient.{IZkStateListener, ZkClient, ZkConnection} import java.net.InetAddress @@ -36,16 +36,24 @@ import java.net.InetAddress class KafkaHealthcheck(private val brokerId: Int, private val advertisedEndpoints: Map[SecurityProtocol, EndPoint], private val zkSessionTimeoutMs: Int, - private val zkClient: ZkClient) extends Logging { + private val zkClient: ZkClient, + private val zkConnection: ZkConnection) extends Logging { val brokerIdPath = ZkUtils.BrokerIdsPath + "/" + brokerId val sessionExpireListener = new SessionExpireListener + var zkWatchedEphemeral: ZKWatchedEphemeral = null def startup() { zkClient.subscribeStateChanges(sessionExpireListener) register() } + def shutdown() { + if(zkWatchedEphemeral != null) { + zkWatchedEphemeral.halt + zkWatchedEphemeral = null + } + } /** * Register this broker as "alive" in zookeeper */ @@ -62,7 +70,7 @@ class KafkaHealthcheck(private val brokerId: Int, // only PLAINTEXT is supported as default // if the broker doesn't listen on PLAINTEXT protocol, an empty endpoint will be registered and older clients will be unable to connect val plaintextEndpoint = updatedEndpoints.getOrElse(SecurityProtocol.PLAINTEXT, new EndPoint(null,-1,null)) - ZkUtils.registerBrokerInZk(zkClient, brokerId, plaintextEndpoint.host, plaintextEndpoint.port, updatedEndpoints, zkSessionTimeoutMs, jmxPort) + zkWatchedEphemeral = ZkUtils.registerBrokerInZk(zkClient, zkConnection, brokerId, plaintextEndpoint.host, plaintextEndpoint.port, updatedEndpoints, zkSessionTimeoutMs, jmxPort) } /** @@ -73,6 +81,8 @@ class KafkaHealthcheck(private val brokerId: Int, @throws(classOf[Exception]) def handleStateChanged(state: KeeperState) { // do nothing, since zkclient will do reconnect for us. + if(state == KeeperState.Expired) + shutdown() } /** diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 17db4fa3c3a14..3388629c7395e 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -35,7 +35,7 @@ import org.apache.kafka.common.metrics.{JmxReporter, Metrics} import org.apache.kafka.common.utils.AppInfoParser import scala.collection.mutable -import org.I0Itec.zkclient.ZkClient +import org.I0Itec.zkclient.{ZkClient, ZkConnection} import kafka.controller.{ControllerStats, KafkaController} import kafka.cluster.{EndPoint, Broker} import kafka.api.{ControlledShutdownResponse, ControlledShutdownRequest} @@ -122,6 +122,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg val metadataCache: MetadataCache = new MetadataCache(config.brokerId) var zkClient: ZkClient = null + var zkConnection : ZkConnection = null val correlationId: AtomicInteger = new AtomicInteger(0) val brokerMetaPropsFile = "meta.properties" val brokerMetadataCheckpoints = config.logDirs.map(logDir => (logDir, new BrokerMetadataCheckpoint(new File(logDir + File.separator +brokerMetaPropsFile)))).toMap @@ -157,7 +158,9 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg kafkaScheduler.startup() /* setup zookeeper */ - zkClient = initZk() + var (client, connection) = initZk() + zkClient = client + zkConnection = connection /* start log manager */ logManager = createLogManager(zkClient, brokerState) @@ -175,7 +178,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg replicaManager.startup() /* start kafka controller */ - kafkaController = new KafkaController(config, zkClient, brokerState) + kafkaController = new KafkaController(config, zkClient, zkConnection, brokerState) kafkaController.startup() /* start kafka coordinator */ @@ -203,7 +206,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg else (protocol, endpoint) } - kafkaHealthcheck = new KafkaHealthcheck(config.brokerId, listeners, config.zkSessionTimeoutMs, zkClient) + kafkaHealthcheck = new KafkaHealthcheck(config.brokerId, listeners, config.zkSessionTimeoutMs, zkClient, zkConnection) kafkaHealthcheck.startup() /* register broker metrics */ @@ -225,7 +228,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg } } - private def initZk(): ZkClient = { + private def initZk(): (ZkClient, ZkConnection) = { info("Connecting to zookeeper on " + config.zkConnect) val chroot = { @@ -243,9 +246,9 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime) extends Logg zkClientForChrootCreation.close() } - val zkClient = ZkUtils.createZkClient(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + val (zkClient, zkConnection) = ZkUtils.createZkClientAndConnection(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) ZkUtils.setupCommonPaths(zkClient) - zkClient + (zkClient, zkConnection) } diff --git a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala index a5c5fb3a9385d..949e8959cf190 100755 --- a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala +++ b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala @@ -18,7 +18,7 @@ package kafka.server import kafka.utils.ZkUtils._ import kafka.utils.CoreUtils._ -import kafka.utils.{Json, SystemTime, Logging} +import kafka.utils.{Json, SystemTime, Logging, ZKWatchedEphemeral} import org.I0Itec.zkclient.exception.ZkNodeExistsException import org.I0Itec.zkclient.IZkDataListener import kafka.controller.ControllerContext @@ -56,7 +56,8 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, case None => -1 } } - + + var zkWatchedEphemeral : ZKWatchedEphemeral = null; def elect: Boolean = { val timestamp = SystemTime.milliseconds.toString val electString = Json.encode(Map("version" -> 1, "brokerid" -> brokerId, "timestamp" -> timestamp)) @@ -73,9 +74,14 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, } try { - createEphemeralPathExpectConflictHandleZKBug(controllerContext.zkClient, electionPath, electString, brokerId, - (controllerString : String, leaderId : Any) => KafkaController.parseControllerId(controllerString) == leaderId.asInstanceOf[Int], - controllerContext.zkSessionTimeout) + //createEphemeralPathExpectConflictHandleZKBug(controllerContext.zkClient, electionPath, electString, brokerId, + // (controllerString : String, leaderId : Any) => KafkaController.parseControllerId(controllerString) == leaderId.asInstanceOf[Int], + // controllerContext.zkSessionTimeout) + assert(zkWatchedEphemeral == null) + zkWatchedEphemeral = new ZKWatchedEphemeral(electionPath, + electString, + controllerContext.zkConnection.getZookeeper) + zkWatchedEphemeral.createAndWatch info(brokerId + " successfully elected as leader") leaderId = brokerId onBecomingLeader() @@ -104,6 +110,10 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, def resign() = { leaderId = -1 + if(zkWatchedEphemeral != null) { + zkWatchedEphemeral.halt + zkWatchedEphemeral = null + } deletePath(controllerContext.zkClient, electionPath) } diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 2ede77f84db17..6b328062e921d 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -204,23 +204,27 @@ object ZkUtils extends Logging { * @param timeout * @param jmxPort */ - def registerBrokerInZk(zkClient: ZkClient, id: Int, host: String, port: Int, advertisedEndpoints: immutable.Map[SecurityProtocol, EndPoint], timeout: Int, jmxPort: Int) { + def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, id: Int, host: String, port: Int, advertisedEndpoints: immutable.Map[SecurityProtocol, EndPoint], timeout: Int, jmxPort: Int): ZKWatchedEphemeral = { val brokerIdPath = ZkUtils.BrokerIdsPath + "/" + id val timestamp = SystemTime.milliseconds.toString val brokerInfo = Json.encode(Map("version" -> 2, "host" -> host, "port" -> port, "endpoints"->advertisedEndpoints.values.map(_.connectionString).toArray, "jmx_port" -> jmxPort, "timestamp" -> timestamp)) val expectedBroker = new Broker(id, advertisedEndpoints) - - registerBrokerInZk(zkClient, brokerIdPath, brokerInfo, expectedBroker, timeout) + val zkWatchedEphemeral = registerBrokerInZk(zkClient, zkConnection, brokerIdPath, brokerInfo, expectedBroker, timeout) info("Registered broker %d at path %s with addresses: %s".format(id, brokerIdPath, advertisedEndpoints.mkString(","))) + zkWatchedEphemeral } - private def registerBrokerInZk(zkClient: ZkClient, brokerIdPath: String, brokerInfo: String, expectedBroker: Broker, timeout: Int) { + private def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, brokerIdPath: String, brokerInfo: String, expectedBroker: Broker, timeout: Int): ZKWatchedEphemeral = { try { - createEphemeralPathExpectConflictHandleZKBug(zkClient, brokerIdPath, brokerInfo, expectedBroker, - (brokerString: String, broker: Any) => Broker.createBroker(broker.asInstanceOf[Broker].id, brokerString).equals(broker.asInstanceOf[Broker]), - timeout) + //createEphemeralPathExpectConflictHandleZKBug(zkClient, brokerIdPath, brokerInfo, expectedBroker, + // (brokerString: String, broker: Any) => Broker.createBroker(broker.asInstanceOf[Broker].id, brokerString).equals(broker.asInstanceOf[Broker]), + // timeout) + val zkWatchedEphemeral = new ZKWatchedEphemeral(brokerIdPath, brokerInfo, + zkConnection.getZookeeper) + zkWatchedEphemeral.createAndWatch + zkWatchedEphemeral } catch { case e: ZkNodeExistsException => diff --git a/core/src/test/scala/unit/kafka/admin/AdminTest.scala b/core/src/test/scala/unit/kafka/admin/AdminTest.scala index 9bd8171f484c1..2d18069884d18 100755 --- a/core/src/test/scala/unit/kafka/admin/AdminTest.scala +++ b/core/src/test/scala/unit/kafka/admin/AdminTest.scala @@ -66,7 +66,7 @@ class AdminTest extends ZooKeeperTestHarness with Logging { @Test def testManualReplicaAssignment() { val brokers = List(0, 1, 2, 3, 4) - TestUtils.createBrokersInZk(zkClient, brokers) + TestUtils.createBrokersInZk(zkClient, zkConnection, brokers) // duplicate brokers intercept[IllegalArgumentException] { @@ -117,7 +117,7 @@ class AdminTest extends ZooKeeperTestHarness with Logging { 11 -> 1 ) val topic = "test" - TestUtils.createBrokersInZk(zkClient, List(0, 1, 2, 3, 4)) + TestUtils.createBrokersInZk(zkClient, zkConnection, List(0, 1, 2, 3, 4)) // create the topic AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topic, expectedReplicaAssignment) // create leaders for all partitions @@ -137,7 +137,7 @@ class AdminTest extends ZooKeeperTestHarness with Logging { def testTopicCreationWithCollision() { val topic = "test.topic" val collidingTopic = "test_topic" - TestUtils.createBrokersInZk(zkClient, List(0, 1, 2, 3, 4)) + TestUtils.createBrokersInZk(zkClient, zkConnection, List(0, 1, 2, 3, 4)) // create the topic AdminUtils.createTopic(zkClient, topic, 3, 1) diff --git a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala index 9bfec72261eaf..d4fa0d5c1b58a 100644 --- a/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/TopicCommandTest.scala @@ -36,7 +36,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging { val cleanupVal = "compact" // create brokers val brokers = List(0, 1, 2) - TestUtils.createBrokersInZk(zkClient, brokers) + TestUtils.createBrokersInZk(zkClient, zkConnection, brokers) // create the topic val createOpts = new TopicCommandOptions(Array("--partitions", numPartitionsOriginal.toString, "--replication-factor", "1", @@ -67,7 +67,7 @@ class TopicCommandTest extends ZooKeeperTestHarness with Logging { // create brokers val brokers = List(0, 1, 2) - TestUtils.createBrokersInZk(zkClient, brokers) + TestUtils.createBrokersInZk(zkClient, zkConnection, brokers) // create the NormalTopic val createOpts = new TopicCommandOptions(Array("--partitions", numPartitionsOriginal.toString, diff --git a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala index bb12a50f6c903..1cf399fea9084 100755 --- a/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala +++ b/core/src/test/scala/unit/kafka/server/LeaderElectionTest.scala @@ -126,7 +126,7 @@ class LeaderElectionTest extends ZooKeeperTestHarness { val brokers = servers.map(s => new Broker(s.config.brokerId, "localhost", s.boundPort())) val brokerEndPoints = brokers.map(b => b.getBrokerEndPoint(SecurityProtocol.PLAINTEXT)) - val controllerContext = new ControllerContext(zkClient, 6000) + val controllerContext = new ControllerContext(zkClient, zkConnection, 6000) controllerContext.liveBrokers = brokers.toSet val controllerChannelManager = new ControllerChannelManager(controllerContext, controllerConfig) controllerChannelManager.startup() diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index 00fbb61077a0b..92d1b4b6b27ec 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -31,7 +31,7 @@ import org.apache.kafka.common.utils.Utils._ import collection.mutable.ListBuffer -import org.I0Itec.zkclient.ZkClient +import org.I0Itec.zkclient.{ZkClient, ZkConnection} import kafka.server._ import kafka.producer._ @@ -506,9 +506,9 @@ object TestUtils extends Logging { } } - def createBrokersInZk(zkClient: ZkClient, ids: Seq[Int]): Seq[Broker] = { + def createBrokersInZk(zkClient: ZkClient, zkConnection: ZkConnection, ids: Seq[Int]): Seq[Broker] = { val brokers = ids.map(id => new Broker(id, "localhost", 6667, SecurityProtocol.PLAINTEXT)) - brokers.foreach(b => ZkUtils.registerBrokerInZk(zkClient, b.id, "localhost", 6667, b.endPoints, 6000, jmxPort = -1)) + brokers.foreach(b => ZkUtils.registerBrokerInZk(zkClient, zkConnection, b.id, "localhost", 6667, b.endPoints, 6000, jmxPort = -1)) brokers } diff --git a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala index e4bfb48c2602d..8adf6affdda27 100755 --- a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala +++ b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala @@ -17,7 +17,7 @@ package kafka.zk -import org.I0Itec.zkclient.ZkClient +import org.I0Itec.zkclient.{ZkClient, ZkConnection} import kafka.utils.{ZkUtils, CoreUtils} import org.junit.{After, Before} import org.scalatest.junit.JUnitSuite @@ -26,6 +26,7 @@ trait ZooKeeperTestHarness extends JUnitSuite { var zkPort: Int = -1 var zookeeper: EmbeddedZookeeper = null var zkClient: ZkClient = null + var zkConnection : ZkConnection = null val zkConnectionTimeout = 6000 val zkSessionTimeout = 6000 @@ -35,7 +36,9 @@ trait ZooKeeperTestHarness extends JUnitSuite { def setUp() { zookeeper = new EmbeddedZookeeper() zkPort = zookeeper.port - zkClient = ZkUtils.createZkClient(zkConnect, zkSessionTimeout, zkConnectionTimeout) + var (client, connection) = ZkUtils.createZkClientAndConnection(zkConnect, zkSessionTimeout, zkConnectionTimeout) + zkClient = client + zkConnection = connection } @After From 4c83ac2609ed29a0f1887bf5087dab50e3e93488 Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Sat, 29 Aug 2015 16:07:23 +0100 Subject: [PATCH 12/25] KAFKA-1387: Removing whitespaces. --- core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala | 2 +- core/src/main/scala/kafka/utils/ZkUtils.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala index 949e8959cf190..4d79b3e26d217 100755 --- a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala +++ b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala @@ -56,7 +56,7 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, case None => -1 } } - + var zkWatchedEphemeral : ZKWatchedEphemeral = null; def elect: Boolean = { val timestamp = SystemTime.milliseconds.toString diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 6b328062e921d..8bd969406c7aa 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -944,9 +944,9 @@ class ZKWatchedEphemeral(path : String, info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) } } - } + } } - + private class EphemeralWatcher extends Watcher { def process(event : WatchedEvent) { // if node deleted, then recreate it From af28ca11647ab81ff88a2ead4d74d960c8af479e Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Wed, 2 Sep 2015 11:16:42 +0100 Subject: [PATCH 13/25] KAFKA-1387: Added checker back and other improvements. --- .../consumer/ZookeeperConsumerConnector.scala | 9 ++- .../kafka/server/ZookeeperLeaderElector.scala | 2 + core/src/main/scala/kafka/utils/ZkUtils.scala | 80 +++++++++++++++++-- 3 files changed, 81 insertions(+), 10 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index 8f9ad5b5ecd0a..af957209c4a8f 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -40,9 +40,6 @@ import org.I0Itec.zkclient.exception.ZkNodeExistsException import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection} import org.apache.zookeeper.Watcher.Event.KeeperState -import scala.collection._ -import scala.collection.JavaConversions._ - /** * This class handles the consumers interaction with zookeeper @@ -275,9 +272,13 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, assert(zkWatchedEphemeral == null) zkWatchedEphemeral = new ZKWatchedEphemeral(dirs. - consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, + consumerRegistryDir + "/" + consumerIdString, + consumerRegistrationInfo, + null, + (consumerZKString, consumer) => true, zkConnection.getZookeeper) zkWatchedEphemeral.createAndWatch + info("end registering consumer " + consumerIdString + " in ZK") } diff --git a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala index 4d79b3e26d217..58af1a0a8b21b 100755 --- a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala +++ b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala @@ -80,6 +80,8 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, assert(zkWatchedEphemeral == null) zkWatchedEphemeral = new ZKWatchedEphemeral(electionPath, electString, + brokerId, + (controllerString : String, leaderId : Any) => KafkaController.parseControllerId(controllerString) == leaderId.asInstanceOf[Int], controllerContext.zkConnection.getZookeeper) zkWatchedEphemeral.createAndWatch info(brokerId + " successfully elected as leader") diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 8bd969406c7aa..658d5ee51e9f2 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -17,11 +17,12 @@ package kafka.utils +import java.util.concurrent.CountDownLatch import kafka.cluster._ import kafka.consumer.{ConsumerThreadId, TopicCount} import kafka.server.ConfigType import org.I0Itec.zkclient.{ZkClient,ZkConnection} -import org.I0Itec.zkclient.exception.{ZkNodeExistsException, ZkNoNodeException, +import org.I0Itec.zkclient.exception.{ZkException, ZkNodeExistsException, ZkNoNodeException, ZkMarshallingError, ZkBadVersionException} import org.I0Itec.zkclient.serialize.ZkSerializer import org.apache.kafka.common.config.ConfigException @@ -36,15 +37,16 @@ import kafka.controller.KafkaController import kafka.controller.LeaderIsrAndControllerEpoch import kafka.common.TopicAndPartition -import org.apache.zookeeper.AsyncCallback.StatCallback -import org.apache.zookeeper.AsyncCallback.StringCallback +import org.apache.zookeeper.AsyncCallback.{DataCallback,StatCallback,StringCallback} import org.apache.zookeeper.CreateMode +import org.apache.zookeeper.KeeperException import org.apache.zookeeper.KeeperException.Code import org.apache.zookeeper.WatchedEvent import org.apache.zookeeper.Watcher import org.apache.zookeeper.ZooDefs.Ids import org.apache.zookeeper.ZooKeeper + object ZkUtils extends Logging { val ConsumersPath = "/consumers" val BrokerIdsPath = "/brokers/ids" @@ -221,11 +223,13 @@ object ZkUtils extends Logging { //createEphemeralPathExpectConflictHandleZKBug(zkClient, brokerIdPath, brokerInfo, expectedBroker, // (brokerString: String, broker: Any) => Broker.createBroker(broker.asInstanceOf[Broker].id, brokerString).equals(broker.asInstanceOf[Broker]), // timeout) - val zkWatchedEphemeral = new ZKWatchedEphemeral(brokerIdPath, brokerInfo, + val zkWatchedEphemeral = new ZKWatchedEphemeral(brokerIdPath, + brokerInfo, + expectedBroker, + (brokerString: String, broker: Any) => Broker.createBroker(broker.asInstanceOf[Broker].id, brokerString).equals(broker.asInstanceOf[Broker]), zkConnection.getZookeeper) zkWatchedEphemeral.createAndWatch zkWatchedEphemeral - } catch { case e: ZkNodeExistsException => throw new RuntimeException("A broker is already registered on the path " + brokerIdPath @@ -914,10 +918,15 @@ object ZkPath { class ZKWatchedEphemeral(path : String, data : String, + expectedCallerData: Any, + checker: (String, Any) => Boolean, zkHandle : ZooKeeper) extends Logging { private val createCallback = new CreateCallback private val ephemeralWatcher = new EphemeralWatcher private val existsCallback = new ExistsCallback + private val getDataCallback = new GetDataCallback + val latch: CountDownLatch = new CountDownLatch(1) + var result: Code = Code.OK @volatile private var stop : Boolean = false private class CreateCallback extends StringCallback { @@ -925,10 +934,13 @@ class ZKWatchedEphemeral(path : String, path : String, ctx : Object, name : String) { + info("### Return code for the create: " + Code.get(rc)) Code.get(rc) match { case Code.OK => { // check that exists and wait checkAndWatch + info("#### Created the znode correctly") + setResult(Code.OK) } case Code.CONNECTIONLOSS => { // try again @@ -936,17 +948,51 @@ class ZKWatchedEphemeral(path : String, } case Code.NONODE => { error("No node for path %s (could be the parent missing)".format(path)) + setResult(Code.NONODE) + } + case Code.NODEEXISTS => { + zkHandle.getData(path, false, getDataCallback, null) } case Code.SESSIONEXPIRED => { error("Session has expired while creating %s".format(path)) } case _ => { info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + setResult(Code.get(rc)) } } } } + private class GetDataCallback extends DataCallback { + def processResult(rc : Int, + path : String, + ctx : Object, + readData: Array[Byte], + stat : Stat) { + Code.get(rc) match { + case Code.OK => { + if (checker(ZKStringSerializer.deserialize(readData).asInstanceOf[String], expectedCallerData)) { + info("An ephemeral node [%s] at %s already exists ".format(data, path) + + "it might be because of a past session I have had or connection issues") + checkAndWatch + setResult(Code.OK) + } else { + info("### Someone else created this znode already") + setResult(Code.NODEEXISTS) + } + } + case Code.NONODE => { + info("The ephemeral node [%s] at %s has gone away while reading it, ".format(data, path)) + setResult(Code.NONODE) + } + case _ => { + setResult(Code.get(rc)) + } + } + } + } + private class EphemeralWatcher extends Watcher { def process(event : WatchedEvent) { // if node deleted, then recreate it @@ -987,7 +1033,7 @@ class ZKWatchedEphemeral(path : String, debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) if(suffix.isEmpty()) { zkHandle.create(prefix, - data.getBytes(), + ZKStringSerializer.serialize(data), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL, createCallback, @@ -1016,12 +1062,15 @@ class ZKWatchedEphemeral(path : String, } case Code.NONODE => { error("No node for path %s (could be the parent missing)".format(path)) + setResult(Code.get(rc)) } case Code.SESSIONEXPIRED => { error("Session has expired while creating %s".format(path)) + setResult(Code.get(rc)) } case _ => { info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + setResult(Code.get(rc)) } } } @@ -1040,6 +1089,16 @@ class ZKWatchedEphemeral(path : String, } } + private def setResult(code: Code) { + result = code + latch.countDown() + } + + private def waitUntilResolved(): Code = { + latch.await() + result + } + def createAndWatch() { val index = path.indexOf('/', 1) match { case -1 => path.length @@ -1049,6 +1108,15 @@ class ZKWatchedEphemeral(path : String, val suffix = path.substring(index, path.length) debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) createRecursive(prefix, suffix) + val result = waitUntilResolved() + result match { + case Code.OK => { + // Nothing to do + } + case _ => { + throw ZkException.create(KeeperException.create(result)) + } + } } def halt() { From 3a46b28ac4499e7ce5eddfc3050eb33b89a8abed Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Wed, 2 Sep 2015 11:57:51 +0100 Subject: [PATCH 14/25] KAFKA-1387: Removed messages used for local debugging. --- core/src/main/scala/kafka/utils/ZkUtils.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 658d5ee51e9f2..ebc1705ebccbe 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -934,12 +934,10 @@ class ZKWatchedEphemeral(path : String, path : String, ctx : Object, name : String) { - info("### Return code for the create: " + Code.get(rc)) Code.get(rc) match { case Code.OK => { // check that exists and wait checkAndWatch - info("#### Created the znode correctly") setResult(Code.OK) } case Code.CONNECTIONLOSS => { @@ -978,7 +976,8 @@ class ZKWatchedEphemeral(path : String, checkAndWatch setResult(Code.OK) } else { - info("### Someone else created this znode already") + info("An ephemeral node [%s] at %s already exists ".format(data, path) + + "and has been created by someone else") setResult(Code.NODEEXISTS) } } From d694fd690b8c43eb1baa5db809b0d669c6b9689f Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Wed, 2 Sep 2015 14:07:01 +0100 Subject: [PATCH 15/25] KAFKA-1387: Re-adding imports removed by mistake. --- .../main/scala/kafka/consumer/ZookeeperConsumerConnector.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index af957209c4a8f..477bf5f163a1a 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -40,6 +40,8 @@ import org.I0Itec.zkclient.exception.ZkNodeExistsException import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection} import org.apache.zookeeper.Watcher.Event.KeeperState +import scala.collection._ +import scala.collection.JavaConversions._ /** * This class handles the consumers interaction with zookeeper From ddc41d96ee5a0e9d9985436c5d9306c82ff1a7f7 Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Tue, 8 Sep 2015 14:50:42 +0100 Subject: [PATCH 16/25] KAFKA-1387: Added test cases and did some general cleanup. --- .../consumer/ZookeeperConsumerConnector.scala | 4 - core/src/main/scala/kafka/utils/ZkUtils.scala | 53 ++----- .../scala/unit/kafka/zk/ZKEphemeralTest.scala | 142 ++++++++++++++++++ 3 files changed, 151 insertions(+), 48 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index 477bf5f163a1a..ee27dac907d6d 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -268,10 +268,6 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, val timestamp = SystemTime.milliseconds.toString val consumerRegistrationInfo = Json.encode(Map("version" -> 1, "subscription" -> topicCount.getTopicCountMap, "pattern" -> topicCount.pattern, "timestamp" -> timestamp)) - - //createEphemeralPathExpectConflictHandleZKBug(zkClient, dirs.consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, null, - // (consumerZKString, consumer) => true, config.zkSessionTimeoutMs) - assert(zkWatchedEphemeral == null) zkWatchedEphemeral = new ZKWatchedEphemeral(dirs. consumerRegistryDir + "/" + consumerIdString, diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index ebc1705ebccbe..fabb11e911545 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -220,9 +220,6 @@ object ZkUtils extends Logging { private def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, brokerIdPath: String, brokerInfo: String, expectedBroker: Broker, timeout: Int): ZKWatchedEphemeral = { try { - //createEphemeralPathExpectConflictHandleZKBug(zkClient, brokerIdPath, brokerInfo, expectedBroker, - // (brokerString: String, broker: Any) => Broker.createBroker(broker.asInstanceOf[Broker].id, brokerString).equals(broker.asInstanceOf[Broker]), - // timeout) val zkWatchedEphemeral = new ZKWatchedEphemeral(brokerIdPath, brokerInfo, expectedBroker, @@ -318,47 +315,6 @@ object ZkUtils extends Logging { } } - /** - * Create an ephemeral node with the given path and data. - * Throw NodeExistsException if node already exists. - * Handles the following ZK session timeout bug: - * - * https://issues.apache.org/jira/browse/ZOOKEEPER-1740 - * - * Upon receiving a NodeExistsException, read the data from the conflicted path and - * trigger the checker function comparing the read data and the expected data, - * If the checker function returns true then the above bug might be encountered, back off and retry; - * otherwise re-throw the exception - */ - def createEphemeralPathExpectConflictHandleZKBug(zkClient: ZkClient, path: String, data: String, expectedCallerData: Any, checker: (String, Any) => Boolean, backoffTime: Int): Unit = { - while (true) { - try { - createEphemeralPathExpectConflict(zkClient, path, data) - return - } catch { - case e: ZkNodeExistsException => { - // An ephemeral node may still exist even after its corresponding session has expired - // due to a Zookeeper bug, in this case we need to retry writing until the previous node is deleted - // and hence the write succeeds without ZkNodeExistsException - ZkUtils.readDataMaybeNull(zkClient, path)._1 match { - case Some(writtenData) => { - if (checker(writtenData, expectedCallerData)) { - info("I wrote this conflicted ephemeral node [%s] at %s a while back in a different session, ".format(data, path) - + "hence I will backoff for this node to be deleted by Zookeeper and retry") - - Thread.sleep(backoffTime) - } else { - throw e - } - } - case None => // the node disappeared; retry creating the ephemeral node immediately - } - } - case e2: Throwable => throw e2 - } - } - } - /** * Create an persistent node with the given path and data. Create parents if necessary. */ @@ -916,6 +872,15 @@ object ZkPath { } } +/** + * Implements an ephemeral znode that is recreated in the case it disappears. + * A znode can be deleted in this case if it has been created in a previous session. + * Say that the client that created it crashes leaving the session open. In this case, + * this client will try to create it and fail, but eventually the ephemeral znode will + * be remove and needs to be created and associated to this session. + * + * It additionally creates the parent path recursively. + */ class ZKWatchedEphemeral(path : String, data : String, expectedCallerData: Any, diff --git a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala index f240e89b67d3f..082023d2cab60 100644 --- a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala +++ b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala @@ -19,7 +19,13 @@ package kafka.zk import kafka.consumer.ConsumerConfig import kafka.utils.ZkUtils +import kafka.utils.ZKWatchedEphemeral import kafka.utils.TestUtils +import org.apache.zookeeper.CreateMode +import org.apache.zookeeper.WatchedEvent +import org.apache.zookeeper.Watcher +import org.apache.zookeeper.ZooDefs.Ids +import org.I0Itec.zkclient.exception.ZkException import org.junit.{Test, Assert} class ZKEphemeralTest extends ZooKeeperTestHarness { @@ -44,4 +50,140 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { val nodeExists = ZkUtils.pathExists(zkClient, "/tmp/zktest") Assert.assertFalse(nodeExists) } + + /***** + ***** Tests for ZkWatchedEphemeral + *****/ + + /** + * Tests basic creation + */ + @Test + def testZkWatchedEphemeral = { + val path = "/zwe-test" + testCreation(path) + } + + /** + * Tests recursive creation + */ + @Test + def testZkWatchedEphemeralRecursive = { + val path = "/zwe-test-parent/zwe-test" + testCreation(path) + } + + private def testCreation(path: String) { + val zk = zkConnection.getZookeeper + val zwe = new ZKWatchedEphemeral(path,"", "", (String, Any) => true, zk) + var created = false + var counter = 10 + + zk.exists(path, new Watcher() { + def process(event: WatchedEvent) { + if(event.getType == Watcher.Event.EventType.NodeCreated) { + created = true + } + } + }) + zwe.createAndWatch + // Waits until the znode is created + while(!created && counter > 0) { + Thread.sleep(100) + counter = counter - 1 + } + // If the znode hasn't been created within the given time, + // then fail the test + if(counter <= 0) + Assert.fail("Failed to create ephemeral znode") + } + + /** + * Tests that it verifies existing and expected data appropriately + */ + @Test + def testExpectedData = { + val path = "/zwe-test" + val zk = zkConnection.getZookeeper + // Try same data + var zwe = new ZKWatchedEphemeral(path, + "test", + "test", + (data: String, expected: Any) => data.equals(expected.asInstanceOf[String]), + zk) + try{ + zwe.createAndWatch + } catch { + case e: ZkException => { + Assert.fail("Shouldn't have thrown a ZK exception %s".format(e.toString)) + } + case e: Throwable => { + Assert.fail("Unexpected exception %s".format(e.toString)) + } + } + // Try different data + zwe = new ZKWatchedEphemeral(path, + "test", + "test-diff", + (data: String, expected: Any) => data.equals(expected.asInstanceOf[String]), + zk) + try{ + zwe.createAndWatch + Assert.fail("Expected a ZkException") + } catch { + case e: ZkException => { + // Expected exception + } + case e: Throwable => { + Assert.fail("Unexpected exception %s".format(e.toString)) + } + } + } + + /** + * Tests that it recreates the znode in the presence of an overlapping + * session. + */ + @Test + def testOverlappingSessions = { + val path = "/zwe-test" + val zk1 = zkConnection.getZookeeper + + //Creates a second session + val (zkClient2, zkConnection2) = ZkUtils.createZkClientAndConnection(zkConnect, zkSessionTimeoutMs, zkConnectionTimeout) + val zk2 = zkConnection2.getZookeeper + val zwe = new ZKWatchedEphemeral(path,"", "", (String, Any) => true, zk2) + val numIterations = 10 + var counter = numIterations + var deleted = false + + // Creates znode for path in the first session + zk1.create(path, Array[Byte](), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL) + // Watches the znode + zk2.exists(path, new Watcher() { + def process(event: WatchedEvent) { + if(event.getType == Watcher.Event.EventType.NodeDeleted) { + deleted = true + } + } + }) + //Bootstraps the ZKWatchedEphemeral object + zwe.createAndWatch + //Closes the first session + zkClient.close + while(!deleted && counter > 0) { + Thread.sleep(100) + counter = counter - 1 + } + counter = numIterations + // Waits until the znode is recreated + while((zk2.exists(path, false) == null) && counter > 0) { + Thread.sleep(100) + counter = counter - 1 + } + // If the znode hasn't been created within the given time, + // then fail the test + if(counter <= 0) + Assert.fail("Failed to create ephemeral znode") + } } From ed7345aec4188ccf4872b3833d8f9b163a037157 Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Tue, 8 Sep 2015 16:42:53 +0100 Subject: [PATCH 17/25] KAFKA-1387: Removed whitespace. --- core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala index 082023d2cab60..a7a671eaafe15 100644 --- a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala +++ b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala @@ -50,7 +50,7 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { val nodeExists = ZkUtils.pathExists(zkClient, "/tmp/zktest") Assert.assertFalse(nodeExists) } - + /***** ***** Tests for ZkWatchedEphemeral *****/ From 2b2ad5b00486377c40d7e0abedf36425277b5315 Mon Sep 17 00:00:00 2001 From: flavio junqueira Date: Fri, 11 Sep 2015 16:28:20 +0100 Subject: [PATCH 18/25] KAFKA-1387: Adding debug. --- core/src/main/scala/kafka/utils/ZkUtils.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index fabb11e911545..b2c27e1959475 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -935,7 +935,7 @@ class ZKWatchedEphemeral(path : String, stat : Stat) { Code.get(rc) match { case Code.OK => { - if (checker(ZKStringSerializer.deserialize(readData).asInstanceOf[String], expectedCallerData)) { + if (checker(ZKStringSerializer.deserialize(readData).asInstanceOf[String], expectedCallerData)) { info("An ephemeral node [%s] at %s already exists ".format(data, path) + "it might be because of a past session I have had or connection issues") checkAndWatch @@ -1073,6 +1073,7 @@ class ZKWatchedEphemeral(path : String, debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) createRecursive(prefix, suffix) val result = waitUntilResolved() + info("Result of znode creation is: %s".format(result)) result match { case Code.OK => { // Nothing to do From 8f351b34cbfdb7d4aab2f03bd2610ececaadb11d Mon Sep 17 00:00:00 2001 From: fpj Date: Sat, 19 Sep 2015 09:49:18 -0700 Subject: [PATCH 19/25] KAFKA-1387: Added debug messages --- .../main/scala/kafka/utils/ShutdownableThread.scala | 12 ++++++++---- .../scala/unit/kafka/server/ServerShutdownTest.scala | 9 +++++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/kafka/utils/ShutdownableThread.scala b/core/src/main/scala/kafka/utils/ShutdownableThread.scala index dc467975ff18d..70c311fa90e43 100644 --- a/core/src/main/scala/kafka/utils/ShutdownableThread.scala +++ b/core/src/main/scala/kafka/utils/ShutdownableThread.scala @@ -26,7 +26,8 @@ abstract class ShutdownableThread(val name: String, val isInterruptible: Boolean this.logIdent = "[" + name + "], " val isRunning: AtomicBoolean = new AtomicBoolean(true) private val shutdownLatch = new CountDownLatch(1) - + (new Exception).printStackTrace() + def shutdown() = { initiateShutdown() awaitShutdown() @@ -47,8 +48,9 @@ abstract class ShutdownableThread(val name: String, val isInterruptible: Boolean * After calling initiateShutdown(), use this API to wait until the shutdown is complete */ def awaitShutdown(): Unit = { + info("About to complete shutdown: %s".format(this.getClass.getCanonicalName)) shutdownLatch.await() - info("Shutdown completed") + info("Shutdown completed: %s".format(this.getClass.getCanonicalName)) } /** @@ -57,15 +59,17 @@ abstract class ShutdownableThread(val name: String, val isInterruptible: Boolean def doWork(): Unit override def run(): Unit = { - info("Starting ") + info("Starting %s %d".format(name, this.getId)) try{ while(isRunning.get()){ doWork() } } catch{ - case e: Throwable => + case e: Throwable => { if(isRunning.get()) error("Error due to ", e) + info("### Got interrupted: %s %d".format(name, this.getId)) + } } shutdownLatch.countDown() info("Stopped ") diff --git a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala index 102dba95290ab..3dd1357c1eb80 100755 --- a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala +++ b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala @@ -149,8 +149,13 @@ class ServerShutdownTest extends ZooKeeperTestHarness { val threadName = Option(t.getClass.getCanonicalName) .getOrElse(t.getClass.getName()) .toLowerCase - - !t.isDaemon && t.isAlive && threadName.startsWith("kafka") + val retValue = !t.isDaemon && t.isAlive && threadName.startsWith("kafka") + if(retValue) { + info("### Thread name: %s %s".format(threadName, t.getStackTrace)) + t.interrupt() + } + + retValue } def verifyNonDaemonThreadsStatus() { From f2e738f644e265476f06805861354394e88ecef8 Mon Sep 17 00:00:00 2001 From: fpj Date: Sun, 20 Sep 2015 08:43:09 -0700 Subject: [PATCH 20/25] KAFKA-1387: Updating lines added for test debugging. --- .../main/scala/kafka/utils/ShutdownableThread.scala | 13 ++++--------- .../unit/kafka/server/ServerShutdownTest.scala | 7 +------ 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/core/src/main/scala/kafka/utils/ShutdownableThread.scala b/core/src/main/scala/kafka/utils/ShutdownableThread.scala index 70c311fa90e43..59c0de3c26c0f 100644 --- a/core/src/main/scala/kafka/utils/ShutdownableThread.scala +++ b/core/src/main/scala/kafka/utils/ShutdownableThread.scala @@ -26,8 +26,7 @@ abstract class ShutdownableThread(val name: String, val isInterruptible: Boolean this.logIdent = "[" + name + "], " val isRunning: AtomicBoolean = new AtomicBoolean(true) private val shutdownLatch = new CountDownLatch(1) - (new Exception).printStackTrace() - + def shutdown() = { initiateShutdown() awaitShutdown() @@ -48,9 +47,8 @@ abstract class ShutdownableThread(val name: String, val isInterruptible: Boolean * After calling initiateShutdown(), use this API to wait until the shutdown is complete */ def awaitShutdown(): Unit = { - info("About to complete shutdown: %s".format(this.getClass.getCanonicalName)) shutdownLatch.await() - info("Shutdown completed: %s".format(this.getClass.getCanonicalName)) + info("Shutdown completed") } /** @@ -65,11 +63,8 @@ abstract class ShutdownableThread(val name: String, val isInterruptible: Boolean doWork() } } catch{ - case e: Throwable => { - if(isRunning.get()) - error("Error due to ", e) - info("### Got interrupted: %s %d".format(name, this.getId)) - } + case e: Throwable => + error("Error due to ", e) } shutdownLatch.countDown() info("Stopped ") diff --git a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala index 3dd1357c1eb80..a28e2ba60c9b6 100755 --- a/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala +++ b/core/src/test/scala/unit/kafka/server/ServerShutdownTest.scala @@ -149,13 +149,8 @@ class ServerShutdownTest extends ZooKeeperTestHarness { val threadName = Option(t.getClass.getCanonicalName) .getOrElse(t.getClass.getName()) .toLowerCase - val retValue = !t.isDaemon && t.isAlive && threadName.startsWith("kafka") - if(retValue) { - info("### Thread name: %s %s".format(threadName, t.getStackTrace)) - t.interrupt() - } - retValue + !t.isDaemon && t.isAlive && threadName.startsWith("kafka") } def verifyNonDaemonThreadsStatus() { From 422f320785ea4e8d04dca064e97143039aa30cde Mon Sep 17 00:00:00 2001 From: Flavio Junqueira Date: Tue, 22 Sep 2015 11:09:49 -0700 Subject: [PATCH 21/25] KAFKA-1387: Addressing first batch of comments. --- .../scala/kafka/consumer/ZookeeperConsumerConnector.scala | 6 +++--- core/src/main/scala/kafka/server/KafkaHealthcheck.scala | 5 ++--- core/src/main/scala/kafka/server/KafkaServer.scala | 2 +- .../main/scala/kafka/server/ZookeeperLeaderElector.scala | 5 +---- core/src/main/scala/kafka/utils/ZkUtils.scala | 2 +- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index ee27dac907d6d..04272191f462e 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -179,7 +179,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, private def connectZk() { info("Connecting to zookeeper instance at " + config.zkConnect) - var (client, connection) = ZkUtils.createZkClientAndConnection(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) + val (client, connection) = ZkUtils.createZkClientAndConnection(config.zkConnect, config.zkSessionTimeoutMs, config.zkConnectionTimeoutMs) zkClient = client zkConnection = connection } @@ -215,7 +215,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, if (config.autoCommitEnable) commitOffsets(true) if (zkClient != null) { - if(zkWatchedEphemeral != null) { + if (zkWatchedEphemeral != null) { zkWatchedEphemeral.halt zkWatchedEphemeral = null } @@ -262,7 +262,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, // this API is used by unit tests only def getTopicRegistry: Pool[String, Pool[Int, PartitionTopicInfo]] = topicRegistry - private var zkWatchedEphemeral : ZKWatchedEphemeral = null + private var zkWatchedEphemeral: ZKWatchedEphemeral = null private def registerConsumerInZK(dirs: ZKGroupDirs, consumerIdString: String, topicCount: TopicCount) { info("begin registering consumer " + consumerIdString + " in ZK") val timestamp = SystemTime.milliseconds.toString diff --git a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala index 5f3e722308db5..eb98c97990d4b 100644 --- a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala +++ b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala @@ -49,7 +49,7 @@ class KafkaHealthcheck(private val brokerId: Int, } def shutdown() { - if(zkWatchedEphemeral != null) { + if (zkWatchedEphemeral != null) { zkWatchedEphemeral.halt zkWatchedEphemeral = null } @@ -80,8 +80,7 @@ class KafkaHealthcheck(private val brokerId: Int, class SessionExpireListener() extends IZkStateListener { @throws(classOf[Exception]) def handleStateChanged(state: KeeperState) { - // do nothing, since zkclient will do reconnect for us. - if(state == KeeperState.Expired) + if (state == KeeperState.Expired) shutdown() } diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 3a38b527d7178..b49a142a2d212 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -165,7 +165,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr kafkaScheduler.startup() /* setup zookeeper */ - var (client, connection) = initZk() + val (client, connection) = initZk() zkClient = client zkConnection = connection diff --git a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala index e02f595706325..54cb076038e08 100755 --- a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala +++ b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala @@ -74,9 +74,6 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, } try { - //createEphemeralPathExpectConflictHandleZKBug(controllerContext.zkClient, electionPath, electString, brokerId, - // (controllerString : String, leaderId : Any) => KafkaController.parseControllerId(controllerString) == leaderId.asInstanceOf[Int], - // controllerContext.zkSessionTimeout) assert(zkWatchedEphemeral == null) zkWatchedEphemeral = new ZKWatchedEphemeral(electionPath, electString, @@ -112,7 +109,7 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, def resign() = { leaderId = -1 - if(zkWatchedEphemeral != null) { + if (zkWatchedEphemeral != null) { zkWatchedEphemeral.halt zkWatchedEphemeral = null } diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index b2c27e1959475..a1120c11fdbd7 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -980,7 +980,7 @@ class ZKWatchedEphemeral(path : String, error("Session has expired while creating %s".format(path)) } case _ => { - info("ZooKeeper event while checking if registration node exists %s %s".format(path, Code.get(rc))) + info("ZooKeeper event while checking if registration node exists %s (return code %s)".format(path, Code.get(rc))) } } } From b7e45e7cc03c9d96253a0e0f4c609867d9b031cd Mon Sep 17 00:00:00 2001 From: Flavio Junqueira Date: Tue, 22 Sep 2015 23:45:44 -0700 Subject: [PATCH 22/25] KAFKA-1387: Simplified the whole mechanism, removing the watch and relying on the session id for conflicts. --- .../consumer/ZookeeperConsumerConnector.scala | 14 +- .../scala/kafka/server/KafkaHealthcheck.scala | 11 +- .../kafka/server/ZookeeperLeaderElector.scala | 14 +- core/src/main/scala/kafka/utils/ZkUtils.scala | 142 +++++------------- .../scala/unit/kafka/zk/ZKEphemeralTest.scala | 98 +++++------- 5 files changed, 84 insertions(+), 195 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index 04272191f462e..99248c2dde728 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -34,7 +34,7 @@ import kafka.network.BlockingChannel import kafka.serializer._ import kafka.utils.CoreUtils.inLock import kafka.utils.ZkUtils._ -import kafka.utils.ZKWatchedEphemeral +import kafka.utils.ZKCheckedEphemeral import kafka.utils._ import org.I0Itec.zkclient.exception.ZkNodeExistsException import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection} @@ -215,10 +215,6 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, if (config.autoCommitEnable) commitOffsets(true) if (zkClient != null) { - if (zkWatchedEphemeral != null) { - zkWatchedEphemeral.halt - zkWatchedEphemeral = null - } zkClient.close() zkClient = null } @@ -262,20 +258,16 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, // this API is used by unit tests only def getTopicRegistry: Pool[String, Pool[Int, PartitionTopicInfo]] = topicRegistry - private var zkWatchedEphemeral: ZKWatchedEphemeral = null private def registerConsumerInZK(dirs: ZKGroupDirs, consumerIdString: String, topicCount: TopicCount) { info("begin registering consumer " + consumerIdString + " in ZK") val timestamp = SystemTime.milliseconds.toString val consumerRegistrationInfo = Json.encode(Map("version" -> 1, "subscription" -> topicCount.getTopicCountMap, "pattern" -> topicCount.pattern, "timestamp" -> timestamp)) - assert(zkWatchedEphemeral == null) - zkWatchedEphemeral = new ZKWatchedEphemeral(dirs. + val zkWatchedEphemeral = new ZKCheckedEphemeral(dirs. consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, - null, - (consumerZKString, consumer) => true, zkConnection.getZookeeper) - zkWatchedEphemeral.createAndWatch + zkWatchedEphemeral.create info("end registering consumer " + consumerIdString + " in ZK") } diff --git a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala index eb98c97990d4b..18ea97d1c7935 100644 --- a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala +++ b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala @@ -41,19 +41,14 @@ class KafkaHealthcheck(private val brokerId: Int, val brokerIdPath = ZkUtils.BrokerIdsPath + "/" + brokerId val sessionExpireListener = new SessionExpireListener - var zkWatchedEphemeral: ZKWatchedEphemeral = null def startup() { zkClient.subscribeStateChanges(sessionExpireListener) register() } - def shutdown() { - if (zkWatchedEphemeral != null) { - zkWatchedEphemeral.halt - zkWatchedEphemeral = null - } - } + def shutdown() {} + /** * Register this broker as "alive" in zookeeper */ @@ -70,7 +65,7 @@ class KafkaHealthcheck(private val brokerId: Int, // only PLAINTEXT is supported as default // if the broker doesn't listen on PLAINTEXT protocol, an empty endpoint will be registered and older clients will be unable to connect val plaintextEndpoint = updatedEndpoints.getOrElse(SecurityProtocol.PLAINTEXT, new EndPoint(null,-1,null)) - zkWatchedEphemeral = ZkUtils.registerBrokerInZk(zkClient, zkConnection, brokerId, plaintextEndpoint.host, plaintextEndpoint.port, updatedEndpoints, zkSessionTimeoutMs, jmxPort) + ZkUtils.registerBrokerInZk(zkClient, zkConnection, brokerId, plaintextEndpoint.host, plaintextEndpoint.port, updatedEndpoints, zkSessionTimeoutMs, jmxPort) } /** diff --git a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala index 54cb076038e08..80b519e472268 100755 --- a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala +++ b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala @@ -18,7 +18,7 @@ package kafka.server import kafka.utils.ZkUtils._ import kafka.utils.CoreUtils._ -import kafka.utils.{Json, SystemTime, Logging, ZKWatchedEphemeral} +import kafka.utils.{Json, SystemTime, Logging, ZKCheckedEphemeral} import org.I0Itec.zkclient.exception.ZkNodeExistsException import org.I0Itec.zkclient.IZkDataListener import kafka.controller.ControllerContext @@ -57,7 +57,6 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, } } - var zkWatchedEphemeral : ZKWatchedEphemeral = null; def elect: Boolean = { val timestamp = SystemTime.milliseconds.toString val electString = Json.encode(Map("version" -> 1, "brokerid" -> brokerId, "timestamp" -> timestamp)) @@ -74,13 +73,10 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, } try { - assert(zkWatchedEphemeral == null) - zkWatchedEphemeral = new ZKWatchedEphemeral(electionPath, + val zkCheckedEphemeral = new ZKCheckedEphemeral(electionPath, electString, - brokerId, - (controllerString : String, leaderId : Any) => KafkaController.parseControllerId(controllerString) == leaderId.asInstanceOf[Int], controllerContext.zkConnection.getZookeeper) - zkWatchedEphemeral.createAndWatch + zkCheckedEphemeral.create info(brokerId + " successfully elected as leader") leaderId = brokerId onBecomingLeader() @@ -109,10 +105,6 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, def resign() = { leaderId = -1 - if (zkWatchedEphemeral != null) { - zkWatchedEphemeral.halt - zkWatchedEphemeral = null - } deletePath(controllerContext.zkClient, electionPath) } diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index a1120c11fdbd7..aaf0261c1d528 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -206,27 +206,23 @@ object ZkUtils extends Logging { * @param timeout * @param jmxPort */ - def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, id: Int, host: String, port: Int, advertisedEndpoints: immutable.Map[SecurityProtocol, EndPoint], timeout: Int, jmxPort: Int): ZKWatchedEphemeral = { + def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, id: Int, host: String, port: Int, advertisedEndpoints: immutable.Map[SecurityProtocol, EndPoint], timeout: Int, jmxPort: Int) { val brokerIdPath = ZkUtils.BrokerIdsPath + "/" + id val timestamp = SystemTime.milliseconds.toString val brokerInfo = Json.encode(Map("version" -> 2, "host" -> host, "port" -> port, "endpoints"->advertisedEndpoints.values.map(_.connectionString).toArray, "jmx_port" -> jmxPort, "timestamp" -> timestamp)) val expectedBroker = new Broker(id, advertisedEndpoints) - val zkWatchedEphemeral = registerBrokerInZk(zkClient, zkConnection, brokerIdPath, brokerInfo, expectedBroker, timeout) + registerBrokerInZk(zkClient, zkConnection, brokerIdPath, brokerInfo, expectedBroker, timeout) info("Registered broker %d at path %s with addresses: %s".format(id, brokerIdPath, advertisedEndpoints.mkString(","))) - zkWatchedEphemeral } - private def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, brokerIdPath: String, brokerInfo: String, expectedBroker: Broker, timeout: Int): ZKWatchedEphemeral = { + private def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, brokerIdPath: String, brokerInfo: String, expectedBroker: Broker, timeout: Int) { try { - val zkWatchedEphemeral = new ZKWatchedEphemeral(brokerIdPath, + val zkCheckedEphemeral = new ZKCheckedEphemeral(brokerIdPath, brokerInfo, - expectedBroker, - (brokerString: String, broker: Any) => Broker.createBroker(broker.asInstanceOf[Broker].id, brokerString).equals(broker.asInstanceOf[Broker]), zkConnection.getZookeeper) - zkWatchedEphemeral.createAndWatch - zkWatchedEphemeral + zkCheckedEphemeral.create } catch { case e: ZkNodeExistsException => throw new RuntimeException("A broker is already registered on the path " + brokerIdPath @@ -873,56 +869,44 @@ object ZkPath { } /** - * Implements an ephemeral znode that is recreated in the case it disappears. - * A znode can be deleted in this case if it has been created in a previous session. - * Say that the client that created it crashes leaving the session open. In this case, - * this client will try to create it and fail, but eventually the ephemeral znode will - * be remove and needs to be created and associated to this session. - * - * It additionally creates the parent path recursively. + * Creates an ephemeral znode checking the session owner + * in the case of conflict. In the regular case, the + * znode is created and the create call returns OK. If + * the call receives a node exists event, then it checks + * if the session matches. If it does, then it returns OK, + * and otherwise it fails the operation. */ -class ZKWatchedEphemeral(path : String, + +class ZKCheckedEphemeral(path : String, data : String, - expectedCallerData: Any, - checker: (String, Any) => Boolean, zkHandle : ZooKeeper) extends Logging { private val createCallback = new CreateCallback - private val ephemeralWatcher = new EphemeralWatcher - private val existsCallback = new ExistsCallback private val getDataCallback = new GetDataCallback val latch: CountDownLatch = new CountDownLatch(1) var result: Code = Code.OK - @volatile - private var stop : Boolean = false + private class CreateCallback extends StringCallback { def processResult(rc : Int, path : String, ctx : Object, name : String) { Code.get(rc) match { - case Code.OK => { - // check that exists and wait - checkAndWatch + case Code.OK => setResult(Code.OK) - } - case Code.CONNECTIONLOSS => { + case Code.CONNECTIONLOSS => // try again - createAndWatch - } - case Code.NONODE => { + createEphemeral + case Code.NONODE => error("No node for path %s (could be the parent missing)".format(path)) setResult(Code.NONODE) - } - case Code.NODEEXISTS => { + case Code.NODEEXISTS => zkHandle.getData(path, false, getDataCallback, null) - } - case Code.SESSIONEXPIRED => { + case Code.SESSIONEXPIRED => error("Session has expired while creating %s".format(path)) - } - case _ => { + setResult(Code.SESSIONEXPIRED) + case _ => info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) setResult(Code.get(rc)) - } } } } @@ -934,21 +918,14 @@ class ZKWatchedEphemeral(path : String, readData: Array[Byte], stat : Stat) { Code.get(rc) match { - case Code.OK => { - if (checker(ZKStringSerializer.deserialize(readData).asInstanceOf[String], expectedCallerData)) { - info("An ephemeral node [%s] at %s already exists ".format(data, path) - + "it might be because of a past session I have had or connection issues") - checkAndWatch - setResult(Code.OK) - } else { - info("An ephemeral node [%s] at %s already exists ".format(data, path) - + "and has been created by someone else") - setResult(Code.NODEEXISTS) - } - } + case Code.OK => + if (stat.getEphemeralOwner != zkHandle.getSessionId) + setResult(Code.NODEEXISTS) + else + setResult(Code.OK) case Code.NONODE => { - info("The ephemeral node [%s] at %s has gone away while reading it, ".format(data, path)) - setResult(Code.NONODE) + warn("The ephemeral node [%s] at %s has gone away while reading it, ".format(data, path)) + createEphemeral } case _ => { setResult(Code.get(rc)) @@ -956,52 +933,20 @@ class ZKWatchedEphemeral(path : String, } } } - - private class EphemeralWatcher extends Watcher { - def process(event : WatchedEvent) { - // if node deleted, then recreate it - if(!stop && event.getType == Watcher.Event.EventType.NodeDeleted) - createAndWatch - } - } - - private class ExistsCallback extends StatCallback { - def processResult(rc : Int, - path : String, - ctx : Object, - stat : Stat) { - Code.get(rc) match { - case Code.OK => {} - case Code.CONNECTIONLOSS => { - // Backoff and try again - checkAndWatch - } - case Code.SESSIONEXPIRED => { - error("Session has expired while creating %s".format(path)) - } - case _ => { - info("ZooKeeper event while checking if registration node exists %s (return code %s)".format(path, Code.get(rc))) - } - } - } - } - - private def checkAndWatch() { - zkHandle.exists(path, - ephemeralWatcher, - existsCallback, - null) - } - - private def createRecursive(prefix : String, suffix : String) { - debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) - if(suffix.isEmpty()) { - zkHandle.create(prefix, + + private def createEphemeral() { + zkHandle.create(path, ZKStringSerializer.serialize(data), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL, createCallback, null) + } + + private def createRecursive(prefix : String, suffix : String) { + debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) + if(suffix.isEmpty()) { + createEphemeral } else { zkHandle.create(prefix, new Array[Byte](0), @@ -1013,10 +958,7 @@ class ZKWatchedEphemeral(path : String, ctx : Object, name : String) { Code.get(rc) match { - case Code.OK => { - // Nothing to do - } - case Code.NODEEXISTS => { + case Code.OK | Code.NODEEXISTS => { // Nothing to do } case Code.CONNECTIONLOSS => { @@ -1063,7 +1005,7 @@ class ZKWatchedEphemeral(path : String, result } - def createAndWatch() { + def create() { val index = path.indexOf('/', 1) match { case -1 => path.length case x : Int => x @@ -1083,8 +1025,4 @@ class ZKWatchedEphemeral(path : String, } } } - - def halt() { - stop = true - } } diff --git a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala index a7a671eaafe15..acc63bba07b40 100644 --- a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala +++ b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala @@ -19,13 +19,13 @@ package kafka.zk import kafka.consumer.ConsumerConfig import kafka.utils.ZkUtils -import kafka.utils.ZKWatchedEphemeral +import kafka.utils.ZKCheckedEphemeral import kafka.utils.TestUtils import org.apache.zookeeper.CreateMode import org.apache.zookeeper.WatchedEvent import org.apache.zookeeper.Watcher import org.apache.zookeeper.ZooDefs.Ids -import org.I0Itec.zkclient.exception.ZkException +import org.I0Itec.zkclient.exception.{ZkException,ZkNodeExistsException} import org.junit.{Test, Assert} class ZKEphemeralTest extends ZooKeeperTestHarness { @@ -75,7 +75,7 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { private def testCreation(path: String) { val zk = zkConnection.getZookeeper - val zwe = new ZKWatchedEphemeral(path,"", "", (String, Any) => true, zk) + val zwe = new ZKCheckedEphemeral(path,"", zk) var created = false var counter = 10 @@ -86,7 +86,7 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { } } }) - zwe.createAndWatch + zwe.create // Waits until the znode is created while(!created && counter > 0) { Thread.sleep(100) @@ -99,49 +99,7 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { } /** - * Tests that it verifies existing and expected data appropriately - */ - @Test - def testExpectedData = { - val path = "/zwe-test" - val zk = zkConnection.getZookeeper - // Try same data - var zwe = new ZKWatchedEphemeral(path, - "test", - "test", - (data: String, expected: Any) => data.equals(expected.asInstanceOf[String]), - zk) - try{ - zwe.createAndWatch - } catch { - case e: ZkException => { - Assert.fail("Shouldn't have thrown a ZK exception %s".format(e.toString)) - } - case e: Throwable => { - Assert.fail("Unexpected exception %s".format(e.toString)) - } - } - // Try different data - zwe = new ZKWatchedEphemeral(path, - "test", - "test-diff", - (data: String, expected: Any) => data.equals(expected.asInstanceOf[String]), - zk) - try{ - zwe.createAndWatch - Assert.fail("Expected a ZkException") - } catch { - case e: ZkException => { - // Expected exception - } - case e: Throwable => { - Assert.fail("Unexpected exception %s".format(e.toString)) - } - } - } - - /** - * Tests that it recreates the znode in the presence of an overlapping + * Tests that it fails in the presence of an overlapping * session. */ @Test @@ -152,7 +110,7 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { //Creates a second session val (zkClient2, zkConnection2) = ZkUtils.createZkClientAndConnection(zkConnect, zkSessionTimeoutMs, zkConnectionTimeout) val zk2 = zkConnection2.getZookeeper - val zwe = new ZKWatchedEphemeral(path,"", "", (String, Any) => true, zk2) + var zwe = new ZKCheckedEphemeral(path,"", zk2) val numIterations = 10 var counter = numIterations var deleted = false @@ -168,22 +126,36 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { } }) //Bootstraps the ZKWatchedEphemeral object - zwe.createAndWatch - //Closes the first session - zkClient.close - while(!deleted && counter > 0) { - Thread.sleep(100) - counter = counter - 1 + var gotException = false; + try { + zwe.create + } catch { + case e: ZkNodeExistsException => + gotException = true } - counter = numIterations - // Waits until the znode is recreated - while((zk2.exists(path, false) == null) && counter > 0) { - Thread.sleep(100) - counter = counter - 1 + Assert.assertTrue(gotException) + } + + /** + * Tests if succeeds with znode from the same session + * + */ + @Test + def testSameSession = { + val path = "/zwe-test" + val zk = zkConnection.getZookeeper + // Creates znode for path in the first session + zk.create(path, Array[Byte](), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL) + + var zwe = new ZKCheckedEphemeral(path,"", zk) + //Bootstraps the ZKWatchedEphemeral object + var gotException = false; + try { + zwe.create + } catch { + case e: ZkNodeExistsException => + gotException = true } - // If the znode hasn't been created within the given time, - // then fail the test - if(counter <= 0) - Assert.fail("Failed to create ephemeral znode") + Assert.assertFalse(gotException) } } From 84cacc2305f5ee1f26bbcddd6a897733c83f844e Mon Sep 17 00:00:00 2001 From: Flavio Junqueira Date: Wed, 23 Sep 2015 17:06:52 -0700 Subject: [PATCH 23/25] KAFKA-1387: Second round of comments addressed. --- .../consumer/ZookeeperConsumerConnector.scala | 4 +- .../scala/kafka/server/KafkaHealthcheck.scala | 9 +- .../kafka/server/ZookeeperLeaderElector.scala | 2 +- .../kafka/utils/ShutdownableThread.scala | 5 +- core/src/main/scala/kafka/utils/ZkUtils.scala | 91 +++++++++---------- .../scala/unit/kafka/utils/TestUtils.scala | 2 +- .../scala/unit/kafka/zk/ZKEphemeralTest.scala | 30 ++---- .../unit/kafka/zk/ZooKeeperTestHarness.scala | 2 +- 8 files changed, 58 insertions(+), 87 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index 99248c2dde728..3a934344812a2 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -34,7 +34,6 @@ import kafka.network.BlockingChannel import kafka.serializer._ import kafka.utils.CoreUtils.inLock import kafka.utils.ZkUtils._ -import kafka.utils.ZKCheckedEphemeral import kafka.utils._ import org.I0Itec.zkclient.exception.ZkNodeExistsException import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener, ZkClient, ZkConnection} @@ -258,6 +257,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, // this API is used by unit tests only def getTopicRegistry: Pool[String, Pool[Int, PartitionTopicInfo]] = topicRegistry + private def registerConsumerInZK(dirs: ZKGroupDirs, consumerIdString: String, topicCount: TopicCount) { info("begin registering consumer " + consumerIdString + " in ZK") val timestamp = SystemTime.milliseconds.toString @@ -267,7 +267,7 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, consumerRegistryDir + "/" + consumerIdString, consumerRegistrationInfo, zkConnection.getZookeeper) - zkWatchedEphemeral.create + zkWatchedEphemeral.create() info("end registering consumer " + consumerIdString + " in ZK") } diff --git a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala index 18ea97d1c7935..688bd24729d46 100644 --- a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala +++ b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala @@ -47,8 +47,6 @@ class KafkaHealthcheck(private val brokerId: Int, register() } - def shutdown() {} - /** * Register this broker as "alive" in zookeeper */ @@ -65,7 +63,7 @@ class KafkaHealthcheck(private val brokerId: Int, // only PLAINTEXT is supported as default // if the broker doesn't listen on PLAINTEXT protocol, an empty endpoint will be registered and older clients will be unable to connect val plaintextEndpoint = updatedEndpoints.getOrElse(SecurityProtocol.PLAINTEXT, new EndPoint(null,-1,null)) - ZkUtils.registerBrokerInZk(zkClient, zkConnection, brokerId, plaintextEndpoint.host, plaintextEndpoint.port, updatedEndpoints, zkSessionTimeoutMs, jmxPort) + ZkUtils.registerBrokerInZk(zkClient, zkConnection, brokerId, plaintextEndpoint.host, plaintextEndpoint.port, updatedEndpoints, jmxPort) } /** @@ -74,10 +72,7 @@ class KafkaHealthcheck(private val brokerId: Int, */ class SessionExpireListener() extends IZkStateListener { @throws(classOf[Exception]) - def handleStateChanged(state: KeeperState) { - if (state == KeeperState.Expired) - shutdown() - } + def handleStateChanged(state: KeeperState) {} /** * Called after the zookeeper session has expired and a new session has been created. You would have to re-create diff --git a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala index 80b519e472268..9844e9ad4a51f 100755 --- a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala +++ b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala @@ -76,7 +76,7 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, val zkCheckedEphemeral = new ZKCheckedEphemeral(electionPath, electString, controllerContext.zkConnection.getZookeeper) - zkCheckedEphemeral.create + zkCheckedEphemeral.create() info(brokerId + " successfully elected as leader") leaderId = brokerId onBecomingLeader() diff --git a/core/src/main/scala/kafka/utils/ShutdownableThread.scala b/core/src/main/scala/kafka/utils/ShutdownableThread.scala index 59c0de3c26c0f..dc467975ff18d 100644 --- a/core/src/main/scala/kafka/utils/ShutdownableThread.scala +++ b/core/src/main/scala/kafka/utils/ShutdownableThread.scala @@ -57,14 +57,15 @@ abstract class ShutdownableThread(val name: String, val isInterruptible: Boolean def doWork(): Unit override def run(): Unit = { - info("Starting %s %d".format(name, this.getId)) + info("Starting ") try{ while(isRunning.get()){ doWork() } } catch{ case e: Throwable => - error("Error due to ", e) + if(isRunning.get()) + error("Error due to ", e) } shutdownLatch.countDown() info("Stopped ") diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index aaf0261c1d528..066759e0215de 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -37,12 +37,10 @@ import kafka.controller.KafkaController import kafka.controller.LeaderIsrAndControllerEpoch import kafka.common.TopicAndPartition -import org.apache.zookeeper.AsyncCallback.{DataCallback,StatCallback,StringCallback} +import org.apache.zookeeper.AsyncCallback.{DataCallback,StringCallback} import org.apache.zookeeper.CreateMode import org.apache.zookeeper.KeeperException import org.apache.zookeeper.KeeperException.Code -import org.apache.zookeeper.WatchedEvent -import org.apache.zookeeper.Watcher import org.apache.zookeeper.ZooDefs.Ids import org.apache.zookeeper.ZooKeeper @@ -206,23 +204,22 @@ object ZkUtils extends Logging { * @param timeout * @param jmxPort */ - def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, id: Int, host: String, port: Int, advertisedEndpoints: immutable.Map[SecurityProtocol, EndPoint], timeout: Int, jmxPort: Int) { + def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, id: Int, host: String, port: Int, advertisedEndpoints: immutable.Map[SecurityProtocol, EndPoint], jmxPort: Int) { val brokerIdPath = ZkUtils.BrokerIdsPath + "/" + id val timestamp = SystemTime.milliseconds.toString val brokerInfo = Json.encode(Map("version" -> 2, "host" -> host, "port" -> port, "endpoints"->advertisedEndpoints.values.map(_.connectionString).toArray, "jmx_port" -> jmxPort, "timestamp" -> timestamp)) - val expectedBroker = new Broker(id, advertisedEndpoints) - registerBrokerInZk(zkClient, zkConnection, brokerIdPath, brokerInfo, expectedBroker, timeout) + registerBrokerInZk(zkClient, zkConnection, brokerIdPath, brokerInfo) info("Registered broker %d at path %s with addresses: %s".format(id, brokerIdPath, advertisedEndpoints.mkString(","))) } - private def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, brokerIdPath: String, brokerInfo: String, expectedBroker: Broker, timeout: Int) { + private def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, brokerIdPath: String, brokerInfo: String) { try { val zkCheckedEphemeral = new ZKCheckedEphemeral(brokerIdPath, brokerInfo, zkConnection.getZookeeper) - zkCheckedEphemeral.create + zkCheckedEphemeral.create() } catch { case e: ZkNodeExistsException => throw new RuntimeException("A broker is already registered on the path " + brokerIdPath @@ -877,19 +874,19 @@ object ZkPath { * and otherwise it fails the operation. */ -class ZKCheckedEphemeral(path : String, - data : String, - zkHandle : ZooKeeper) extends Logging { +class ZKCheckedEphemeral(path: String, + data: String, + zkHandle: ZooKeeper) extends Logging { private val createCallback = new CreateCallback private val getDataCallback = new GetDataCallback val latch: CountDownLatch = new CountDownLatch(1) var result: Code = Code.OK private class CreateCallback extends StringCallback { - def processResult(rc : Int, - path : String, - ctx : Object, - name : String) { + def processResult(rc: Int, + path: String, + ctx: Object, + name: String) { Code.get(rc) match { case Code.OK => setResult(Code.OK) @@ -905,79 +902,75 @@ class ZKCheckedEphemeral(path : String, error("Session has expired while creating %s".format(path)) setResult(Code.SESSIONEXPIRED) case _ => - info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + warn("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) setResult(Code.get(rc)) } } } private class GetDataCallback extends DataCallback { - def processResult(rc : Int, - path : String, - ctx : Object, - readData: Array[Byte], - stat : Stat) { + def processResult(rc: Int, + path: String, + ctx: Object, + readData: Array[Byte], + stat: Stat) { Code.get(rc) match { case Code.OK => if (stat.getEphemeralOwner != zkHandle.getSessionId) setResult(Code.NODEEXISTS) else setResult(Code.OK) - case Code.NONODE => { - warn("The ephemeral node [%s] at %s has gone away while reading it, ".format(data, path)) + case Code.NONODE => + info("The ephemeral node [%s] at %s has gone away while reading it, ".format(data, path)) createEphemeral - } - case _ => { + case Code.SESSIONEXPIRED => + error("Session has expired while reading znode %s".format(path)) + setResult(Code.SESSIONEXPIRED) + case _ => setResult(Code.get(rc)) - } } } } private def createEphemeral() { zkHandle.create(path, - ZKStringSerializer.serialize(data), - Ids.OPEN_ACL_UNSAFE, - CreateMode.EPHEMERAL, - createCallback, - null) + ZKStringSerializer.serialize(data), + Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL, + createCallback, + null) } - private def createRecursive(prefix : String, suffix : String) { + private def createRecursive(prefix: String, suffix: String) { debug("Path: %s, Prefix: %s, Suffix: %s".format(path, prefix, suffix)) if(suffix.isEmpty()) { createEphemeral } else { zkHandle.create(prefix, - new Array[Byte](0), - Ids.OPEN_ACL_UNSAFE, - CreateMode.PERSISTENT, - new StringCallback() { + new Array[Byte](0), + Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT, + new StringCallback() { def processResult(rc : Int, path : String, ctx : Object, name : String) { Code.get(rc) match { - case Code.OK | Code.NODEEXISTS => { + case Code.OK | Code.NODEEXISTS => // Nothing to do - } - case Code.CONNECTIONLOSS => { + case Code.CONNECTIONLOSS => // try again val suffix = ctx.asInstanceOf[String] createRecursive(path, suffix) - } - case Code.NONODE => { + case Code.NONODE => error("No node for path %s (could be the parent missing)".format(path)) setResult(Code.get(rc)) - } - case Code.SESSIONEXPIRED => { + case Code.SESSIONEXPIRED => error("Session has expired while creating %s".format(path)) setResult(Code.get(rc)) - } - case _ => { + case _ => info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) setResult(Code.get(rc)) - } } } }, @@ -1017,12 +1010,10 @@ class ZKCheckedEphemeral(path : String, val result = waitUntilResolved() info("Result of znode creation is: %s".format(result)) result match { - case Code.OK => { + case Code.OK => // Nothing to do - } - case _ => { + case _ => throw ZkException.create(KeeperException.create(result)) - } } } } diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index b85cb951ec894..7f482fb61adcf 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -502,7 +502,7 @@ object TestUtils extends Logging { def createBrokersInZk(zkClient: ZkClient, zkConnection: ZkConnection, ids: Seq[Int]): Seq[Broker] = { val brokers = ids.map(id => new Broker(id, "localhost", 6667, SecurityProtocol.PLAINTEXT)) - brokers.foreach(b => ZkUtils.registerBrokerInZk(zkClient, zkConnection, b.id, "localhost", 6667, b.endPoints, 6000, jmxPort = -1)) + brokers.foreach(b => ZkUtils.registerBrokerInZk(zkClient, zkConnection, b.id, "localhost", 6667, b.endPoints, jmxPort = -1)) brokers } diff --git a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala index acc63bba07b40..8c86a20d32253 100644 --- a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala +++ b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala @@ -86,16 +86,10 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { } } }) - zwe.create + zwe.create() // Waits until the znode is created - while(!created && counter > 0) { - Thread.sleep(100) - counter = counter - 1 - } - // If the znode hasn't been created within the given time, - // then fail the test - if(counter <= 0) - Assert.fail("Failed to create ephemeral znode") + TestUtils.waitUntilTrue(() => ZkUtils.pathExists(zkClient, path), + "Znode %s wasn't created".format(path)) } /** @@ -108,27 +102,17 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { val zk1 = zkConnection.getZookeeper //Creates a second session - val (zkClient2, zkConnection2) = ZkUtils.createZkClientAndConnection(zkConnect, zkSessionTimeoutMs, zkConnectionTimeout) + val (_, zkConnection2) = ZkUtils.createZkClientAndConnection(zkConnect, zkSessionTimeoutMs, zkConnectionTimeout) val zk2 = zkConnection2.getZookeeper var zwe = new ZKCheckedEphemeral(path,"", zk2) - val numIterations = 10 - var counter = numIterations - var deleted = false // Creates znode for path in the first session zk1.create(path, Array[Byte](), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL) - // Watches the znode - zk2.exists(path, new Watcher() { - def process(event: WatchedEvent) { - if(event.getType == Watcher.Event.EventType.NodeDeleted) { - deleted = true - } - } - }) + //Bootstraps the ZKWatchedEphemeral object var gotException = false; try { - zwe.create + zwe.create() } catch { case e: ZkNodeExistsException => gotException = true @@ -151,7 +135,7 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { //Bootstraps the ZKWatchedEphemeral object var gotException = false; try { - zwe.create + zwe.create() } catch { case e: ZkNodeExistsException => gotException = true diff --git a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala index 8adf6affdda27..3e1c6e0ceecc1 100755 --- a/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala +++ b/core/src/test/scala/unit/kafka/zk/ZooKeeperTestHarness.scala @@ -36,7 +36,7 @@ trait ZooKeeperTestHarness extends JUnitSuite { def setUp() { zookeeper = new EmbeddedZookeeper() zkPort = zookeeper.port - var (client, connection) = ZkUtils.createZkClientAndConnection(zkConnect, zkSessionTimeout, zkConnectionTimeout) + val (client, connection) = ZkUtils.createZkClientAndConnection(zkConnect, zkSessionTimeout, zkConnectionTimeout) zkClient = client zkConnection = connection } From 390fc11e27ca55902f7c05b4ae1328ec839d9542 Mon Sep 17 00:00:00 2001 From: Flavio Junqueira Date: Thu, 24 Sep 2015 09:47:46 -0700 Subject: [PATCH 24/25] KAFKA-1387: Third round of comments. --- core/src/main/scala/kafka/server/KafkaHealthcheck.scala | 1 - core/src/main/scala/kafka/server/KafkaServer.scala | 2 +- core/src/main/scala/kafka/utils/ZkUtils.scala | 5 +++-- core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala index 688bd24729d46..16760d4f6d3e6 100644 --- a/core/src/main/scala/kafka/server/KafkaHealthcheck.scala +++ b/core/src/main/scala/kafka/server/KafkaHealthcheck.scala @@ -35,7 +35,6 @@ import java.net.InetAddress */ class KafkaHealthcheck(private val brokerId: Int, private val advertisedEndpoints: Map[SecurityProtocol, EndPoint], - private val zkSessionTimeoutMs: Int, private val zkClient: ZkClient, private val zkConnection: ZkConnection) extends Logging { diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index b49a142a2d212..a879417ae8699 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -223,7 +223,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr else (protocol, endpoint) } - kafkaHealthcheck = new KafkaHealthcheck(config.brokerId, listeners, config.zkSessionTimeoutMs, zkClient, zkConnection) + kafkaHealthcheck = new KafkaHealthcheck(config.brokerId, listeners, zkClient, zkConnection) kafkaHealthcheck.startup() /* register broker metrics */ diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 066759e0215de..84000ee894bfb 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -902,7 +902,7 @@ class ZKCheckedEphemeral(path: String, error("Session has expired while creating %s".format(path)) setResult(Code.SESSIONEXPIRED) case _ => - warn("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + warn("ZooKeeper event while creating registration node: %s %s".format(path, Code.get(rc))) setResult(Code.get(rc)) } } @@ -927,6 +927,7 @@ class ZKCheckedEphemeral(path: String, error("Session has expired while reading znode %s".format(path)) setResult(Code.SESSIONEXPIRED) case _ => + warn("ZooKeeper event while getting znode data: %s %s".format(path, Code.get(rc))) setResult(Code.get(rc)) } } @@ -969,7 +970,7 @@ class ZKCheckedEphemeral(path: String, error("Session has expired while creating %s".format(path)) setResult(Code.get(rc)) case _ => - info("ZooKeeper event while creating registration node %s %s".format(path, Code.get(rc))) + warn("ZooKeeper event while creating registration node: %s %s".format(path, Code.get(rc))) setResult(Code.get(rc)) } } diff --git a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala index 8c86a20d32253..267c9f2c26405 100644 --- a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala +++ b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala @@ -104,7 +104,7 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { //Creates a second session val (_, zkConnection2) = ZkUtils.createZkClientAndConnection(zkConnect, zkSessionTimeoutMs, zkConnectionTimeout) val zk2 = zkConnection2.getZookeeper - var zwe = new ZKCheckedEphemeral(path,"", zk2) + var zwe = new ZKCheckedEphemeral(path, "", zk2) // Creates znode for path in the first session zk1.create(path, Array[Byte](), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL) From 7a17280a445a85bf6ff832b8eef5960f9fcfd6c0 Mon Sep 17 00:00:00 2001 From: Flavio Junqueira Date: Thu, 24 Sep 2015 09:56:54 -0700 Subject: [PATCH 25/25] KAFKA-1387: More spaces removed and parameters aligned. --- .../scala/kafka/consumer/ZookeeperConsumerConnector.scala | 6 +++--- core/src/main/scala/kafka/controller/KafkaController.scala | 2 +- core/src/main/scala/kafka/server/KafkaServer.scala | 2 +- .../main/scala/kafka/server/ZookeeperLeaderElector.scala | 4 ++-- core/src/main/scala/kafka/utils/ZkUtils.scala | 2 +- core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala index 3a934344812a2..2027ec8aadadf 100755 --- a/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala +++ b/core/src/main/scala/kafka/consumer/ZookeeperConsumerConnector.scala @@ -264,9 +264,9 @@ private[kafka] class ZookeeperConsumerConnector(val config: ConsumerConfig, val consumerRegistrationInfo = Json.encode(Map("version" -> 1, "subscription" -> topicCount.getTopicCountMap, "pattern" -> topicCount.pattern, "timestamp" -> timestamp)) val zkWatchedEphemeral = new ZKCheckedEphemeral(dirs. - consumerRegistryDir + "/" + consumerIdString, - consumerRegistrationInfo, - zkConnection.getZookeeper) + consumerRegistryDir + "/" + consumerIdString, + consumerRegistrationInfo, + zkConnection.getZookeeper) zkWatchedEphemeral.create() info("end registering consumer " + consumerIdString + " in ZK") diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala index d4098ec9278eb..a7b44cab501a9 100755 --- a/core/src/main/scala/kafka/controller/KafkaController.scala +++ b/core/src/main/scala/kafka/controller/KafkaController.scala @@ -44,7 +44,7 @@ import kafka.server._ import kafka.common.TopicAndPartition class ControllerContext(val zkClient: ZkClient, - val zkConnection : ZkConnection, + val zkConnection: ZkConnection, val zkSessionTimeout: Int) { var controllerChannelManager: ControllerChannelManager = null val controllerLock: ReentrantLock = new ReentrantLock() diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index a879417ae8699..ba3333ddee24b 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -129,7 +129,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = SystemTime, threadNamePr val metadataCache: MetadataCache = new MetadataCache(config.brokerId) var zkClient: ZkClient = null - var zkConnection : ZkConnection = null + var zkConnection: ZkConnection = null val correlationId: AtomicInteger = new AtomicInteger(0) val brokerMetaPropsFile = "meta.properties" val brokerMetadataCheckpoints = config.logDirs.map(logDir => (logDir, new BrokerMetadataCheckpoint(new File(logDir + File.separator +brokerMetaPropsFile)))).toMap diff --git a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala index 9844e9ad4a51f..b283e0a37b410 100755 --- a/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala +++ b/core/src/main/scala/kafka/server/ZookeeperLeaderElector.scala @@ -74,8 +74,8 @@ class ZookeeperLeaderElector(controllerContext: ControllerContext, try { val zkCheckedEphemeral = new ZKCheckedEphemeral(electionPath, - electString, - controllerContext.zkConnection.getZookeeper) + electString, + controllerContext.zkConnection.getZookeeper) zkCheckedEphemeral.create() info(brokerId + " successfully elected as leader") leaderId = brokerId diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 84000ee894bfb..e1cfa2e795964 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -216,7 +216,7 @@ object ZkUtils extends Logging { private def registerBrokerInZk(zkClient: ZkClient, zkConnection: ZkConnection, brokerIdPath: String, brokerInfo: String) { try { - val zkCheckedEphemeral = new ZKCheckedEphemeral(brokerIdPath, + val zkCheckedEphemeral = new ZKCheckedEphemeral(brokerIdPath, brokerInfo, zkConnection.getZookeeper) zkCheckedEphemeral.create() diff --git a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala index 267c9f2c26405..2bf658c5f244c 100644 --- a/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala +++ b/core/src/test/scala/unit/kafka/zk/ZKEphemeralTest.scala @@ -75,7 +75,7 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { private def testCreation(path: String) { val zk = zkConnection.getZookeeper - val zwe = new ZKCheckedEphemeral(path,"", zk) + val zwe = new ZKCheckedEphemeral(path, "", zk) var created = false var counter = 10 @@ -131,7 +131,7 @@ class ZKEphemeralTest extends ZooKeeperTestHarness { // Creates znode for path in the first session zk.create(path, Array[Byte](), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL) - var zwe = new ZKCheckedEphemeral(path,"", zk) + var zwe = new ZKCheckedEphemeral(path, "", zk) //Bootstraps the ZKWatchedEphemeral object var gotException = false; try {