diff --git a/docs/configuration/cluster_manager/cluster_stats.rst b/docs/configuration/cluster_manager/cluster_stats.rst index 4a6e69645a06b..737746ffabc22 100644 --- a/docs/configuration/cluster_manager/cluster_stats.rst +++ b/docs/configuration/cluster_manager/cluster_stats.rst @@ -15,6 +15,7 @@ Every cluster has a statistics tree rooted at *cluster..* with the followi upstream_cx_http2_total, Counter, Total HTTP/2 connections upstream_cx_connect_fail, Counter, Total connection failures upstream_cx_connect_timeout, Counter, Total connection timeouts + upstream_cx_overflow, Counter, Total times that the cluster's connection circuit breaker overflowed upstream_cx_connect_ms, Timer, Connection establishment milliseconds upstream_cx_length_ms, Timer, Connection length milliseconds upstream_cx_destroy, Counter, Total destroyed connections diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index e706a1d16d6de..50192c8d2e06e 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -140,6 +140,7 @@ class HostSet { COUNTER(upstream_cx_http2_total) \ COUNTER(upstream_cx_connect_fail) \ COUNTER(upstream_cx_connect_timeout) \ + COUNTER(upstream_cx_overflow) \ TIMER (upstream_cx_connect_ms) \ TIMER (upstream_cx_length_ms) \ COUNTER(upstream_cx_destroy) \ diff --git a/source/common/filter/tcp_proxy.cc b/source/common/filter/tcp_proxy.cc index 12be486010d39..4d8524ba95a08 100644 --- a/source/common/filter/tcp_proxy.cc +++ b/source/common/filter/tcp_proxy.cc @@ -51,6 +51,7 @@ void TcpProxy::initializeUpstreamConnection() { ->resourceManager(Upstream::ResourcePriority::Default); if (!upstream_cluster_resource_manager.connections().canCreate()) { + cluster_manager_.get(config_->clusterName())->stats().upstream_cx_overflow_.inc(); read_callbacks_->connection().close(Network::ConnectionCloseType::NoFlush); return; } diff --git a/source/common/http/http1/conn_pool.cc b/source/common/http/http1/conn_pool.cc index 5f5f3f49b1a21..03f55766fb76d 100644 --- a/source/common/http/http1/conn_pool.cc +++ b/source/common/http/http1/conn_pool.cc @@ -72,9 +72,14 @@ ConnectionPool::Cancellable* ConnPoolImpl::newStream(StreamDecoder& response_dec } if (host_->cluster().resourceManager(priority_).pendingRequests().canCreate()) { + bool can_create_connection = + host_->cluster().resourceManager(priority_).connections().canCreate(); + if (!can_create_connection) { + host_->cluster().stats().upstream_cx_overflow_.inc(); + } + // If we have no connections at all, make one no matter what so we don't starve. - if ((ready_clients_.size() == 0 && busy_clients_.size() == 0) || - host_->cluster().resourceManager(priority_).connections().canCreate()) { + if ((ready_clients_.size() == 0 && busy_clients_.size() == 0) || can_create_connection) { createNewConnection(); } diff --git a/test/common/filter/tcp_proxy_test.cc b/test/common/filter/tcp_proxy_test.cc index 2d87a2eff2d22..01789890c2a2b 100644 --- a/test/common/filter/tcp_proxy_test.cc +++ b/test/common/filter/tcp_proxy_test.cc @@ -170,6 +170,10 @@ TEST_F(TcpProxyTest, UpstreamConnectionLimit) { // The downstream connection closes if the proxy can't make an upstream connection. EXPECT_CALL(filter_callbacks_.connection_, close(Network::ConnectionCloseType::NoFlush)); ASSERT_EQ(Network::FilterStatus::StopIteration, filter_->onData(buffer)); + EXPECT_EQ( + 1U, + cluster_manager_.cluster_.stats_store_.counter("cluster.fake_cluster.upstream_cx_overflow") + .value()); } } // Filter diff --git a/test/common/http/http1/conn_pool_test.cc b/test/common/http/http1/conn_pool_test.cc index 73f5d36bb140c..183bd2e0d017d 100644 --- a/test/common/http/http1/conn_pool_test.cc +++ b/test/common/http/http1/conn_pool_test.cc @@ -349,6 +349,7 @@ TEST_F(Http1ConnPoolImplTest, MaxConnections) { NiceMock outer_decoder2; ConnPoolCallbacks callbacks2; handle = conn_pool_.newStream(outer_decoder2, callbacks2); + EXPECT_EQ(1U, cluster_.stats_.upstream_cx_overflow_.value()); EXPECT_NE(nullptr, handle);