All queries failed when one BE network or disk has issue #4649
Description
Steps to reproduce the behavior:
- Use iptables drop BE brpc port
- query
2020-09-19 17:40:57,418 WARN 429 [BackendServiceProxy.execPlanFragmentAsync():116] Execute plan fragment catch a exception, address=172.26
.34.141:9990
java.lang.RuntimeException: Unable to validate object
at com.baidu.jprotobuf.pbrpc.transport.ChannelPool.getChannel(ChannelPool.java:85) ~[jprotobuf-rpc-core-3.5.21.jar:?]
at com.baidu.jprotobuf.pbrpc.transport.RpcChannel.getConnection(RpcChannel.java:72) ~[jprotobuf-rpc-core-3.5.21.jar:?]
at com.baidu.jprotobuf.pbrpc.client.ProtobufRpcProxy.invoke(ProtobufRpcProxy.java:486) ~[jprotobuf-rpc-core-3.5.21.jar:?]
at com.sun.proxy.$Proxy35.execPlanFragmentAsync(Unknown Source) ~[?:?]
at org.apache.doris.rpc.BackendServiceProxy.execPlanFragmentAsync(BackendServiceProxy.java:99) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.Coordinator$BackendExecState.execRemoteFragmentAsync(Coordinator.java:1471) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.Coordinator.exec(Coordinator.java:488) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.StmtExecutor.handleQueryStmt(StmtExecutor.java:608) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:254) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:209) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.ConnectProcessor.dispatch(ConnectProcessor.java:343) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.ConnectProcessor.processOnce(ConnectProcessor.java:528) ~[dorisdb-fe.jar:?]
at org.apache.doris.mysql.nio.ReadListener.lambda$handleEvent$0(ReadListener.java:50) ~[dorisdb-fe.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_252]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_252]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_252]
Caused by: java.util.NoSuchElementException: Unable to validate object
at org.apache.commons.pool2.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:502) ~[commons-pool2-2.2.jar:2.2]
at org.apache.commons.pool2.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:361) ~[commons-pool2-2.2.jar:2.2]
at com.baidu.jprotobuf.pbrpc.transport.ChannelPool.getChannel(ChannelPool.java:79) ~[jprotobuf-rpc-core-3.5.21.jar:?]
... 15 more
2020-09-19 17:40:57,418 INFO 429 [QeProcessorImpl.unregisterQuery():78] deregister query id 9c01fd8356c146a9-8e6a454bcf396e02
2020-09-19 17:40:57,419 WARN 429 [StmtExecutor.execute():318] execute Exception
java.lang.NullPointerException: null
at org.apache.doris.qe.Coordinator$BackendExecState$1.get(Coordinator.java:1495) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.Coordinator$BackendExecState$1.get(Coordinator.java:1505) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.Coordinator$BackendExecState$1.get(Coordinator.java:1475) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.Coordinator.exec(Coordinator.java:496) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.StmtExecutor.handleQueryStmt(StmtExecutor.java:608) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:254) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:209) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.ConnectProcessor.dispatch(ConnectProcessor.java:343) ~[dorisdb-fe.jar:?]
at org.apache.doris.qe.ConnectProcessor.processOnce(ConnectProcessor.java:528) ~[dorisdb-fe.jar:?]
at org.apache.doris.mysql.nio.ReadListener.lambda$handleEvent$0(ReadListener.java:50) ~[dorisdb-fe.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_252]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_252]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_252]