Another day, another Apache Curator bug

We just ran into this in production: https://issues.apache.org/jira/browse/CURATOR-358

Long story short, zookeeper was down, druid came up, zookeeper came up, but no master ever took over.

Might want to bump the curator version in the pom.xml.

2017-03-14T20:50:58,718 WARN [qtp997219303-85] org.eclipse.jetty.servlet.ServletHandler - /druid/coordinator/v1/datasources
java.lang.RuntimeException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /druid/prod/coordinator/_COORDINATOR
        at com.google.common.base.Throwables.propagate(Throwables.java:160) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at io.druid.server.coordinator.DruidCoordinator.getCurrentLeader(DruidCoordinator.java:347) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at io.druid.server.http.CoordinatorRedirectInfo.getRedirectURL(CoordinatorRedirectInfo.java:49) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHO
T]
        at io.druid.server.http.RedirectFilter.doFilter(RedirectFilter.java:73) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:83) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:364) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1652) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:221) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1125) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1059) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.server.handler.HandlerList.handle(HandlerList.java:52) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:97) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.server.Server.handle(Server.java:497) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:310) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:248) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.io.AbstractConnection$2.run(AbstractConnection.java:540) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:620) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:540) [druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at java.lang.Thread.run(Thread.java:745) [?:1.8.0_25]
Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /druid/prod/coordinator/_COORDINATOR
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:111) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1650) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.apache.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:230) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.apache.curator.framework.imps.GetChildrenBuilderImpl$3.call(GetChildrenBuilderImpl.java:219) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:108) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.apache.curator.framework.imps.GetChildrenBuilderImpl.pathInForeground(GetChildrenBuilderImpl.java:215) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.apache.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:207) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.apache.curator.framework.imps.GetChildrenBuilderImpl.forPath(GetChildrenBuilderImpl.java:40) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.apache.curator.framework.recipes.locks.LockInternals.getSortedChildren(LockInternals.java:151) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.apache.curator.framework.recipes.locks.LockInternals.getParticipantNodes(LockInternals.java:133) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at org.apache.curator.framework.recipes.leader.LeaderLatch.getLeader(LeaderLatch.java:453) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        at io.druid.server.coordinator.DruidCoordinator.getCurrentLeader(DruidCoordinator.java:339) ~[druid-services-0.9.2.1-SNAPSHOT-selfcontained.jar:0.9.2.1-SNAPSHOT]
        ... 22 more

[drcrallen]

Related. #3837

[drcrallen]

Looks like 2.11.2 got bumped to 2.12.0?

That will not happen fast EDIT: was wrong, 2.12.0 was out and not detectable on mobile :-P

[gianm]

Looks like a version with the fix was released a couple of weeks ago. Thanks @Andy256 for the reminder, I raised #4060 to update our version.

[gianm]

Reopening this issue, since the Curator upgrade was reverted in #4103.