make http/2 settings configurable (#988)

DEPRECATED.md

@@ -9,3 +9,4 @@ The following features have been DEPRECATED and will be removed in the specified
   * Config option `statsd_local_udp_port` has been deprecated and has been replaced with
   `statsd_udp_ip_address`.
   * `HttpFilterConfigFactory` filter API has been deprecated in favor of `NamedHttpFilterConfigFactory`.
+  * Config option `http_codec_options` has been deprecated and has been replaced with `http2_settings`.

docs/configuration/cluster_manager/cluster.rst

@@ -19,6 +19,7 @@ Cluster
     "ssl_context": "{...}",
     "features": "...",
     "http_codec_options": "...",
+    "http2_settings": "{...}",
     "dns_refresh_rate_ms": "...",
     "dns_lookup_family": "...",
     "outlier_detection": "{...}"
@@ -137,6 +138,13 @@ http_codec_options
   an HTTP/2 mesh, if it's desired to disable HTTP/2 header compression the *no_compression*
   option should be specified both here as well as in the HTTP connection manager.
 
+.. _config_cluster_manager_cluster_http2_settings:
+
+http2_settings
+  *(optional, object)* Additional HTTP/2 settings that are passed directly to the HTTP/2 codec when
+  initiating HTTP connection pool connections. These are the same options supported in the HTTP connection
+  manager :ref:`http2_settings <config_http_conn_man_http2_settings>` option.
+
 .. _config_cluster_manager_cluster_dns_refresh_rate_ms:
 
 dns_refresh_rate_ms

docs/configuration/http_conn_man/http_conn_man.rst

@@ -20,6 +20,7 @@ HTTP connection manager
       "add_user_agent": "...",
       "tracing": "{...}",
       "http_codec_options": "...",
+      "http2_settings": "{...}",
       "server_name": "...",
       "idle_timeout_s": "...",
       "drain_timeout_ms": "...",
@@ -85,7 +86,9 @@ add_user_agent
 
 .. _config_http_conn_man_http_codec_options:
 
-http_codec_options
+http_codec_options (Warning: DEPRECATED and will be removed in 1.4.0)
+  **DEPRECATED**, use :ref:`http2_settings <config_http_conn_man_http2_settings>` instead.
+
   *(optional, string)* Additional options that are passed directly to the codec. Not all options
   are applicable to all codecs. Possible values are:
 
@@ -97,6 +100,44 @@ http_codec_options
   <config_cluster_manager_cluster_http_codec_options>` option. See the comment there about
   disabling HTTP/2 header compression.
 
+.. _config_http_conn_man_http2_settings:
+
+http2_settings
+  *(optional, object)* Additional HTTP/2 settings that are passed directly to the HTTP/2 codec.
+  Currently supported settings are:
+
+  hpack_table_size
+    *(optional, integer)* `Maximum table size <http://httpwg.org/specs/rfc7541.html#rfc.section.4.2>`_
+    (in octets) that the encoder is permitted to use for
+    the dynamic HPACK table. Valid values range from 0 to 4294967295 (2^32 - 1) and defaults to 4096.
+    0 effectively disables header compression.
+
+  max_concurrent_streams
+    *(optional, integer)* `Maximum concurrent streams 
+    <http://httpwg.org/specs/rfc7540.html#rfc.section.5.1.2>`_
+    allowed for peer on one HTTP/2 connection.
+    Valid values range from 1 to 2147483647 (2^31 - 1) and defaults to 2147483647.
+
+.. _config_http_conn_man_http2_settings_initial_stream_window_size:
+
+  initial_stream_window_size
+    *(optional, integer)* `Initial stream-level flow-control window 
+    <http://httpwg.org/specs/rfc7540.html#rfc.section.6.9.2>`_ size. Valid values range from 65535
+    (2^16 - 1, HTTP/2 default) to 2147483647 (2^31 - 1, HTTP/2 maximum) and defaults to 268435456
+    (256 * 1024 * 1024).
+
+    NOTE: 65535 is the initial window size from HTTP/2 spec. We only support increasing the default window
+    size now, so it's also the minimum.
+
+  initial_connection_window_size
+    *(optional, integer)* Similar to :ref:`initial_stream_window_size 
+    <config_http_conn_man_http2_settings_initial_stream_window_size>`, but for connection-level flow-control
+    window. Currently , this has the same minimum/maximum/default as :ref:`initial_stream_window_size 
+    <config_http_conn_man_http2_settings_initial_stream_window_size>`.
+
+  These are the same options available in the upstream cluster :ref:`http2_settings
+  <config_cluster_manager_cluster_http2_settings>` option.
+
 .. _config_http_conn_man_server_name:
 
 server_name

include/envoy/http/codec.h

@@ -148,11 +148,47 @@ class ConnectionCallbacks {
 };
 
 /**
- * A list of options that can be specified when creating a codec.
+ * HTTP/2 codec settings
  */
-class CodecOptions {
-public:
-  static const uint64_t NoCompression = 0x1;
+struct Http2Settings {
+  // TODO(jwfang): support other HTTP/2 settings
+  uint32_t hpack_table_size_{DEFAULT_HPACK_TABLE_SIZE};
+  uint32_t max_concurrent_streams_{DEFAULT_MAX_CONCURRENT_STREAMS};
+  uint32_t initial_stream_window_size_{DEFAULT_INITIAL_STREAM_WINDOW_SIZE};
+  uint32_t initial_connection_window_size_{DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE};
+
+  // disable HPACK compression
+  static const uint32_t MIN_HPACK_TABLE_SIZE = 0;
+  // initial value from HTTP/2 spec, same as NGHTTP2_DEFAULT_HEADER_TABLE_SIZE from nghttp2
+  static const uint32_t DEFAULT_HPACK_TABLE_SIZE = (1 << 12);
+  // no maximum from HTTP/2 spec, use unsigned 32-bit maximum
+  static const uint32_t MAX_HPACK_TABLE_SIZE = (1UL << 32) - 1;
+
+  // TODO(jwfang): make this 0, the HTTP/2 spec minimum
+  static const uint32_t MIN_MAX_CONCURRENT_STREAMS = 1;
+  // defaults to maximum, same as nghttp2
+  static const uint32_t DEFAULT_MAX_CONCURRENT_STREAMS = (1U << 31) - 1;
+  // no maximum from HTTP/2 spec, total streams is unsigned 32-bit maximum,
+  // one-side (client/server) is half that, and we need to exclude stream 0.
+  // same as NGHTTP2_INITIAL_MAX_CONCURRENT_STREAMS from nghttp2
+  static const uint32_t MAX_MAX_CONCURRENT_STREAMS = (1U << 31) - 1;
+
+  // initial value from HTTP/2 spec, same as NGHTTP2_INITIAL_WINDOW_SIZE from nghttp2
+  // NOTE: we only support increasing window size now, so this is also the minimum
+  // TODO(jwfang): make this 0 to support decrease window size
+  static const uint32_t MIN_INITIAL_STREAM_WINDOW_SIZE = (1 << 16) - 1;
+  // initial value from HTTP/2 spec is 65535, but we want more (256MiB)
+  static const uint32_t DEFAULT_INITIAL_STREAM_WINDOW_SIZE = 256 * 1024 * 1024;
+  // maximum from HTTP/2 spec, same as NGHTTP2_MAX_WINDOW_SIZE from nghttp2
+  static const uint32_t MAX_INITIAL_STREAM_WINDOW_SIZE = (1U << 31) - 1;
+
+  // CONNECTION_WINDOW_SIZE is similar to STREAM_WINDOW_SIZE, but for connection-level window
+  // TODO(jwfang): make this 0 to support decrease window size
+  static const uint32_t MIN_INITIAL_CONNECTION_WINDOW_SIZE = (1 << 16) - 1;
+  // nghttp2's default connection-level window equals to its stream-level,
+  // our default connection-level window also equals to our stream-level
+  static const uint32_t DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE = 256 * 1024 * 1024;
+  static const uint32_t MAX_INITIAL_CONNECTION_WINDOW_SIZE = (1U << 31) - 1;
 };
 
 /**

include/envoy/upstream/BUILD

@@ -77,6 +77,7 @@ envoy_cc_library(
         ":load_balancer_type_interface",
         ":resource_manager_interface",
         "//include/envoy/common:optional",
+        "//include/envoy/http:codec_interface",
         "//include/envoy/network:connection_interface",
         "//include/envoy/ssl:context_interface",
     ],

include/envoy/upstream/upstream.h

@@ -9,6 +9,7 @@
 #include <vector>
 
 #include "envoy/common/optional.h"
+#include "envoy/http/codec.h"
 #include "envoy/network/connection.h"
 #include "envoy/ssl/context.h"
 #include "envoy/upstream/load_balancer_type.h"
@@ -243,10 +244,10 @@ class ClusterInfo {
   virtual uint64_t features() const PURE;
 
   /**
-   * @return uint64_t HTTP codec options for HTTP connections created on behalf of this cluster.
-   *         @see Http::CodecOptions.
+   * @return const Http::Http2Settings& for HTTP/2 connections created on behalf of this cluster.
+   *         @see Http::Http2Settings.
    */
-  virtual uint64_t httpCodecOptions() const PURE;
+  virtual const Http::Http2Settings& http2Settings() const PURE;
 
   /**
    * @return the type of load balancing that the cluster should use.

source/common/http/codec_client.cc

@@ -128,7 +128,7 @@ CodecClientProd::CodecClientProd(Type type, Network::ClientConnectionPtr&& conne
   }
   case Type::HTTP2: {
     codec_.reset(new Http2::ClientConnectionImpl(*connection_, *this, host->cluster().statsScope(),
-                                                 host->cluster().httpCodecOptions()));
+                                                 host->cluster().http2Settings()));
     break;
   }
   }

source/common/http/http2/codec_impl.cc

@@ -506,24 +506,59 @@ void ConnectionImpl::sendPendingFrames() {
   }
 }
 
-void ConnectionImpl::sendSettings(uint64_t codec_options) {
-  std::vector<nghttp2_settings_entry> iv = {
-      {NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, MAX_CONCURRENT_STREAMS},
-      {NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, DEFAULT_WINDOW_SIZE}};
+void ConnectionImpl::sendSettings(const Http2Settings& http2_settings) {
+  ASSERT(http2_settings.hpack_table_size_ <= Http2Settings::MAX_HPACK_TABLE_SIZE);
+  ASSERT(Http2Settings::MIN_MAX_CONCURRENT_STREAMS <= http2_settings.max_concurrent_streams_ &&
+         http2_settings.max_concurrent_streams_ <= Http2Settings::MAX_MAX_CONCURRENT_STREAMS);
+  ASSERT(
+      Http2Settings::MIN_INITIAL_STREAM_WINDOW_SIZE <= http2_settings.initial_stream_window_size_ &&
+      http2_settings.initial_stream_window_size_ <= Http2Settings::MAX_INITIAL_STREAM_WINDOW_SIZE);
+  ASSERT(Http2Settings::MIN_INITIAL_CONNECTION_WINDOW_SIZE <=
+             http2_settings.initial_connection_window_size_ &&
+         http2_settings.initial_connection_window_size_ <=
+             Http2Settings::MAX_INITIAL_CONNECTION_WINDOW_SIZE);
+
+  std::vector<nghttp2_settings_entry> iv;
+
+  if (http2_settings.hpack_table_size_ != NGHTTP2_DEFAULT_HEADER_TABLE_SIZE) {
+    iv.push_back({NGHTTP2_SETTINGS_HEADER_TABLE_SIZE, http2_settings.hpack_table_size_});
+    conn_log_debug("setting HPACK table size to {}", connection_, http2_settings.hpack_table_size_);
+  }
 
-  if (codec_options & CodecOptions::NoCompression) {
-    iv.push_back({NGHTTP2_SETTINGS_HEADER_TABLE_SIZE, 0});
-    conn_log_debug("disabling header compression", connection_);
+  if (http2_settings.max_concurrent_streams_ != NGHTTP2_INITIAL_MAX_CONCURRENT_STREAMS) {
+    iv.push_back({NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, http2_settings.max_concurrent_streams_});
+    conn_log_debug("setting max concurrent streams to {}", connection_,
+                   http2_settings.max_concurrent_streams_);
   }
 
-  int rc = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, &iv[0], iv.size());
-  ASSERT(rc == 0);
-  UNREFERENCED_PARAMETER(rc);
+  if (http2_settings.initial_stream_window_size_ != NGHTTP2_INITIAL_WINDOW_SIZE) {
+    iv.push_back(
+        {NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE, http2_settings.initial_stream_window_size_});
+    conn_log_debug("setting stream-level initial window size to {}", connection_,
+                   http2_settings.initial_stream_window_size_);
+  }
+
+  if (!iv.empty()) {
+    int rc = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, &iv[0], iv.size());
+    ASSERT(rc == 0);
+    UNREFERENCED_PARAMETER(rc);
+  } else {
+    // nghttp2_submit_settings need to be called at least once
+    int rc = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, 0, 0);
+    ASSERT(rc == 0);
+    UNREFERENCED_PARAMETER(rc);
+  }
 
   // Increase connection window size up to our default size.
-  rc = nghttp2_submit_window_update(session_, NGHTTP2_FLAG_NONE, 0,
-                                    DEFAULT_WINDOW_SIZE - NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE);
-  ASSERT(rc == 0);
+  if (http2_settings.initial_connection_window_size_ != NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE) {
+    conn_log_debug("updating connection-level initial window size to {}", connection_,
+                   http2_settings.initial_connection_window_size_);
+    int rc = nghttp2_submit_window_update(session_, NGHTTP2_FLAG_NONE, 0,
+                                          http2_settings.initial_connection_window_size_ -
+                                              NGHTTP2_INITIAL_CONNECTION_WINDOW_SIZE);
+    ASSERT(rc == 0);
+    UNREFERENCED_PARAMETER(rc);
+  }
 }
 
 ConnectionImpl::Http2Callbacks::Http2Callbacks() {
@@ -602,19 +637,19 @@ ConnectionImpl::Http2Options::Http2Options() {
   // Currently we do not do anything with stream priority. Setting the following option prevents
   // nghttp2 from keeping around closed streams for use during stream priority dependency graph
   // calculations. This saves a tremendous amount of memory in cases where there are a large number
-  // of kept alive http/2 connections.
+  // of kept alive HTTP/2 connections.
   nghttp2_option_set_no_closed_streams(options_, 1);
 }
 
 ConnectionImpl::Http2Options::~Http2Options() { nghttp2_option_del(options_); }
 
 ClientConnectionImpl::ClientConnectionImpl(Network::Connection& connection,
                                            ConnectionCallbacks& callbacks, Stats::Scope& stats,
-                                           uint64_t codec_options)
+                                           const Http2Settings& http2_settings)
     : ConnectionImpl(connection, stats), callbacks_(callbacks) {
   nghttp2_session_client_new2(&session_, http2_callbacks_.callbacks(), base(),
                               http2_options_.options());
-  sendSettings(codec_options);
+  sendSettings(http2_settings);
 }
 
 Http::StreamEncoder& ClientConnectionImpl::newStream(StreamDecoder& decoder) {
@@ -648,11 +683,11 @@ int ClientConnectionImpl::onHeader(const nghttp2_frame* frame, HeaderString&& na
 
 ServerConnectionImpl::ServerConnectionImpl(Network::Connection& connection,
                                            Http::ServerConnectionCallbacks& callbacks,
-                                           Stats::Store& stats, uint64_t codec_options)
+                                           Stats::Store& stats, const Http2Settings& http2_settings)
     : ConnectionImpl(connection, stats), callbacks_(callbacks) {
   nghttp2_session_server_new2(&session_, http2_callbacks_.callbacks(), base(),
                               http2_options_.options());
-  sendSettings(codec_options);
+  sendSettings(http2_settings);
 }
 
 int ServerConnectionImpl::onBeginHeaders(const nghttp2_frame* frame) {

source/common/http/http2/codec_impl.h

@@ -32,7 +32,7 @@ const std::string ALPN_STRING = "h2";
 const std::string CLIENT_MAGIC_PREFIX = "PRI * HTTP/2";
 
 /**
- * All stats for the http/2 codec. @see stats_macros.h
+ * All stats for the HTTP/2 codec. @see stats_macros.h
  */
 // clang-format off
 #define ALL_HTTP2_CODEC_STATS(COUNTER)                                                             \
@@ -44,7 +44,7 @@ const std::string CLIENT_MAGIC_PREFIX = "PRI * HTTP/2";
 // clang-format on
 
 /**
- * Wrapper struct for the http/2 codec stats. @see stats_macros.h
+ * Wrapper struct for the HTTP/2 codec stats. @see stats_macros.h
  */
 struct CodecStats {
   ALL_HTTP2_CODEC_STATS(GENERATE_COUNTER_STRUCT)
@@ -82,8 +82,6 @@ class ConnectionImpl : public virtual Connection, Logger::Loggable<Logger::Id::h
   void shutdownNotice() override;
   bool wantsToWrite() override { return nghttp2_session_want_write(session_); }
 
-  static const uint64_t MAX_CONCURRENT_STREAMS = 1024;
-
 protected:
   /**
    * Wrapper for static nghttp2 callback dispatchers.
@@ -195,7 +193,7 @@ class ConnectionImpl : public virtual Connection, Logger::Loggable<Logger::Id::h
   StreamImpl* getStream(int32_t stream_id);
   int saveHeader(const nghttp2_frame* frame, HeaderString&& name, HeaderString&& value);
   void sendPendingFrames();
-  void sendSettings(uint64_t codec_options);
+  void sendSettings(const Http2Settings& http2_settings);
 
   static Http2Callbacks http2_callbacks_;
   static Http2Options http2_options_;
@@ -215,10 +213,6 @@ class ConnectionImpl : public virtual Connection, Logger::Loggable<Logger::Id::h
   ssize_t onSend(const uint8_t* data, size_t length);
   int onStreamClose(int32_t stream_id, uint32_t error_code);
 
-  // For now just set all window sizes (stream and connection) to 256MiB. We can adjust later if
-  // needed.
-  static const uint64_t DEFAULT_WINDOW_SIZE = 256 * 1024 * 1024;
-
   static const std::unique_ptr<const Http::HeaderMap> CONTINUE_HEADER;
 
   Network::Connection& connection_;
@@ -233,7 +227,7 @@ class ConnectionImpl : public virtual Connection, Logger::Loggable<Logger::Id::h
 class ClientConnectionImpl : public ClientConnection, public ConnectionImpl {
 public:
   ClientConnectionImpl(Network::Connection& connection, ConnectionCallbacks& callbacks,
-                       Stats::Scope& stats, uint64_t codec_options);
+                       Stats::Scope& stats, const Http2Settings& http2_settings);
 
   // Http::ClientConnection
   Http::StreamEncoder& newStream(StreamDecoder& response_decoder) override;
@@ -253,7 +247,7 @@ class ClientConnectionImpl : public ClientConnection, public ConnectionImpl {
 class ServerConnectionImpl : public ServerConnection, public ConnectionImpl {
 public:
   ServerConnectionImpl(Network::Connection& connection, ServerConnectionCallbacks& callbacks,
-                       Stats::Store& stats, uint64_t codec_options);
+                       Stats::Store& stats, const Http2Settings& http2_settings);
 
 private:
   // ConnectionImpl

source/common/http/utility.cc

@@ -136,12 +136,34 @@ bool Utility::isInternalRequest(const HeaderMap& headers) {
   return Network::Utility::isInternalAddress(forwarded_for->value().c_str());
 }
 
-uint64_t Utility::parseCodecOptions(const Json::Object& config) {
-  uint64_t ret = 0;
+Http2Settings Utility::parseHttp2Settings(const Json::Object& config) {
+  Http2Settings ret;
+
+  Json::ObjectSharedPtr http2_settings = config.getObject("http2_settings", true);
+  ret.hpack_table_size_ =
+      http2_settings->getInteger("hpack_table_size", Http::Http2Settings::DEFAULT_HPACK_TABLE_SIZE);
+  ret.max_concurrent_streams_ = http2_settings->getInteger(
+      "max_concurrent_streams", Http::Http2Settings::DEFAULT_MAX_CONCURRENT_STREAMS);
+  ret.initial_stream_window_size_ = http2_settings->getInteger(
+      "initial_stream_window_size", Http::Http2Settings::DEFAULT_INITIAL_STREAM_WINDOW_SIZE);
+  ret.initial_connection_window_size_ =
+      http2_settings->getInteger("initial_connection_window_size",
+                                 Http::Http2Settings::DEFAULT_INITIAL_CONNECTION_WINDOW_SIZE);
+
+  // http_codec_options config is DEPRECATED
   std::string options = config.getString("http_codec_options", "");
+  if (options != "") {
+    spdlog::logger& logger = Logger::Registry::getLog(Logger::Id::config);
+    logger.warn("'http_codec_options' is DEPRECATED, please use 'http2_settings' instead");
+  }
+
   for (const std::string& option : StringUtil::split(options, ',')) {
     if (option == "no_compression") {
-      ret |= CodecOptions::NoCompression;
+      if (http2_settings->hasObject("hpack_table_size") && ret.hpack_table_size_ != 0) {
+        throw EnvoyException(
+            "'http_codec_options.no_compression' conflicts with 'http2_settings.hpack_table_size'");
+      }
+      ret.hpack_table_size_ = 0;
     } else {
       throw EnvoyException(fmt::format("unknown http codec option '{}'", option));
     }