Add an HTTP/2 related rate limiting

doc/admin-guide/files/records.yaml.en.rst

@@ -4532,6 +4532,13 @@ HTTP/2 Configuration
    This limit only will be enforced if :ts:cv:`proxy.config.http2.stream_priority_enabled`
    is set to 1.
 
+.. ts:cv:: CONFIG proxy.config.http2.max_rst_stream_frames_per_minute INT 14
+   :reloadable:
+
+   Specifies how many RST_STREAM frames |TS| receives for a minute at maximum.
+   Clients exceeded this limit will be immediately disconnected with an error
+   code of ENHANCE_YOUR_CALM.
+
 .. ts:cv:: CONFIG proxy.config.http2.min_avg_window_update FLOAT 2560.0
    :reloadable:
 

doc/admin-guide/monitoring/statistics/core/http-connection.en.rst

@@ -280,6 +280,13 @@ HTTP/2
    maximum allowed number of priority frames per minute limit which is configured by
    :ts:cv:`proxy.config.http2.max_priority_frames_per_minute`.
 
+.. ts:stat:: global proxy.process.http2.max_rst_stream_frames_per_minute_exceeded integer
+   :type: counter
+
+   Represents the total number of closed HTTP/2 connections for exceeding the
+   maximum allowed number of rst_stream frames per minute limit which is configured by
+   :ts:cv:`proxy.config.http2.max_rst_stream_frames_per_minute`.
+
 .. ts:stat:: global proxy.process.http2.insufficient_avg_window_update integer
    :type: counter
 

iocore/net/P_SNIActionPerformer.h

@@ -130,6 +130,82 @@ class HTTP2InitialWindowSizeIn : public ActionItem
   int value = -1;
 };
 
+class HTTP2MaxSettingsFramesPerMinute : public ActionItem
+{
+public:
+  HTTP2MaxSettingsFramesPerMinute(int value) : value(value) {}
+  ~HTTP2MaxSettingsFramesPerMinute() override {}
+
+  int
+  SNIAction(SSL &ssl, const Context &ctx) const override
+  {
+    if (auto snis = TLSSNISupport::getInstance(&ssl)) {
+      snis->hints_from_sni.http2_max_settings_frames_per_minute = value;
+    }
+    return SSL_TLSEXT_ERR_OK;
+  }
+
+private:
+  int value = -1;
+};
+
+class HTTP2MaxPingFramesPerMinute : public ActionItem
+{
+public:
+  HTTP2MaxPingFramesPerMinute(int value) : value(value) {}
+  ~HTTP2MaxPingFramesPerMinute() override {}
+
+  int
+  SNIAction(SSL &ssl, const Context &ctx) const override
+  {
+    if (auto snis = TLSSNISupport::getInstance(&ssl)) {
+      snis->hints_from_sni.http2_max_ping_frames_per_minute = value;
+    }
+    return SSL_TLSEXT_ERR_OK;
+  }
+
+private:
+  int value = -1;
+};
+
+class HTTP2MaxPriorityFramesPerMinute : public ActionItem
+{
+public:
+  HTTP2MaxPriorityFramesPerMinute(int value) : value(value) {}
+  ~HTTP2MaxPriorityFramesPerMinute() override {}
+
+  int
+  SNIAction(SSL &ssl, const Context &ctx) const override
+  {
+    if (auto snis = TLSSNISupport::getInstance(&ssl)) {
+      snis->hints_from_sni.http2_max_priority_frames_per_minute = value;
+    }
+    return SSL_TLSEXT_ERR_OK;
+  }
+
+private:
+  int value = -1;
+};
+
+class HTTP2MaxRstStreamFramesPerMinute : public ActionItem
+{
+public:
+  HTTP2MaxRstStreamFramesPerMinute(int value) : value(value) {}
+  ~HTTP2MaxRstStreamFramesPerMinute() override {}
+
+  int
+  SNIAction(SSL &ssl, const Context &ctx) const override
+  {
+    if (auto snis = TLSSNISupport::getInstance(&ssl)) {
+      snis->hints_from_sni.http2_max_rst_stream_frames_per_minute = value;
+    }
+    return SSL_TLSEXT_ERR_OK;
+  }
+
+private:
+  int value = -1;
+};
+
 class TunnelDestination : public ActionItem
 {
   // ID of the configured variable. This will be used to know which function

iocore/net/TLSSNISupport.h

@@ -60,6 +60,10 @@ class TLSSNISupport
     std::optional<uint32_t> http2_buffer_water_mark;
     std::optional<uint32_t> server_max_early_data;
     std::optional<uint32_t> http2_initial_window_size_in;
+    std::optional<uint32_t> http2_max_settings_frames_per_minute;
+    std::optional<uint32_t> http2_max_ping_frames_per_minute;
+    std::optional<uint32_t> http2_max_priority_frames_per_minute;
+    std::optional<uint32_t> http2_max_rst_stream_frames_per_minute;
     std::optional<std::string_view> outbound_sni_policy;
   } hints_from_sni;
 

iocore/net/YamlSNIConfig.cc

@@ -161,6 +161,18 @@ YamlSNIConfig::Item::populate_sni_actions(action_vector_t &actions)
   if (http2_initial_window_size_in.has_value()) {
     actions.push_back(std::make_unique<HTTP2InitialWindowSizeIn>(http2_initial_window_size_in.value()));
   }
+  if (http2_max_settings_frames_per_minute.has_value()) {
+    actions.push_back(std::make_unique<HTTP2MaxSettingsFramesPerMinute>(http2_max_settings_frames_per_minute.value()));
+  }
+  if (http2_max_ping_frames_per_minute.has_value()) {
+    actions.push_back(std::make_unique<HTTP2MaxPingFramesPerMinute>(http2_max_ping_frames_per_minute.value()));
+  }
+  if (http2_max_priority_frames_per_minute.has_value()) {
+    actions.push_back(std::make_unique<HTTP2MaxPriorityFramesPerMinute>(http2_max_priority_frames_per_minute.value()));
+  }
+  if (http2_max_rst_stream_frames_per_minute.has_value()) {
+    actions.push_back(std::make_unique<HTTP2MaxRstStreamFramesPerMinute>(http2_max_rst_stream_frames_per_minute.value()));
+  }
 
   actions.push_back(std::make_unique<ServerMaxEarlyData>(server_max_early_data));
   actions.push_back(std::make_unique<SNI_IpAllow>(ip_allow, fqdn));
@@ -204,6 +216,10 @@ std::set<std::string> valid_sni_config_keys = {TS_fqdn,
                                                TS_http2,
                                                TS_http2_buffer_water_mark,
                                                TS_http2_initial_window_size_in,
+                                               TS_http2_max_settings_frames_per_minute,
+                                               TS_http2_max_ping_frames_per_minute,
+                                               TS_http2_max_priority_frames_per_minute,
+                                               TS_http2_max_rst_stream_frames_per_minute,
                                                TS_quic,
                                                TS_ip_allow,
 #if TS_USE_HELLO_CB || defined(OPENSSL_IS_BORINGSSL)
@@ -247,6 +263,18 @@ template <> struct convert<YamlSNIConfig::Item> {
     if (node[TS_http2_initial_window_size_in]) {
       item.http2_initial_window_size_in = node[TS_http2_initial_window_size_in].as<int>();
     }
+    if (node[TS_http2_max_settings_frames_per_minute]) {
+      item.http2_max_settings_frames_per_minute = node[TS_http2_max_settings_frames_per_minute].as<int>();
+    }
+    if (node[TS_http2_max_ping_frames_per_minute]) {
+      item.http2_max_ping_frames_per_minute = node[TS_http2_max_ping_frames_per_minute].as<int>();
+    }
+    if (node[TS_http2_max_priority_frames_per_minute]) {
+      item.http2_max_priority_frames_per_minute = node[TS_http2_max_priority_frames_per_minute].as<int>();
+    }
+    if (node[TS_http2_max_rst_stream_frames_per_minute]) {
+      item.http2_max_rst_stream_frames_per_minute = node[TS_http2_max_rst_stream_frames_per_minute].as<int>();
+    }
     if (node[TS_quic]) {
       item.offer_quic = node[TS_quic].as<bool>();
     }

iocore/net/YamlSNIConfig.h

@@ -64,6 +64,10 @@ TSDECL(valid_tls_version_min_in);
 TSDECL(valid_tls_version_max_in);
 TSDECL(http2);
 TSDECL(http2_buffer_water_mark);
+TSDECL(http2_max_settings_frames_per_minute);
+TSDECL(http2_max_ping_frames_per_minute);
+TSDECL(http2_max_priority_frames_per_minute);
+TSDECL(http2_max_rst_stream_frames_per_minute);
 TSDECL(quic);
 TSDECL(host_sni_policy);
 TSDECL(http2_initial_window_size_in);
@@ -103,6 +107,10 @@ struct YamlSNIConfig {
     int valid_tls_version_max_in = -1;
     std::vector<int> tunnel_alpn{};
     std::optional<int> http2_buffer_water_mark;
+    std::optional<int> http2_max_settings_frames_per_minute;
+    std::optional<int> http2_max_ping_frames_per_minute;
+    std::optional<int> http2_max_priority_frames_per_minute;
+    std::optional<int> http2_max_rst_stream_frames_per_minute;
     uint32_t server_max_early_data = 0;
     std::optional<int> http2_initial_window_size_in;
 

proxy/http2/HTTP2.cc

@@ -480,21 +480,22 @@ uint32_t Http2::initial_window_size_out               = 65535;
 Http2FlowControlPolicy Http2::flow_control_policy_out = Http2FlowControlPolicy::STATIC_SESSION_AND_STATIC_STREAM;
 uint32_t Http2::no_activity_timeout_out               = 120;
 
-float Http2::stream_error_rate_threshold        = 0.1;
-uint32_t Http2::stream_error_sampling_threshold = 10;
-uint32_t Http2::max_settings_per_frame          = 7;
-uint32_t Http2::max_settings_per_minute         = 14;
-uint32_t Http2::max_settings_frames_per_minute  = 14;
-uint32_t Http2::max_ping_frames_per_minute      = 60;
-uint32_t Http2::max_priority_frames_per_minute  = 120;
-float Http2::min_avg_window_update              = 2560.0;
-uint32_t Http2::con_slow_log_threshold          = 0;
-uint32_t Http2::stream_slow_log_threshold       = 0;
-uint32_t Http2::header_table_size_limit         = 65536;
-uint32_t Http2::write_buffer_block_size         = 262144;
-float Http2::write_size_threshold               = 0.5;
-uint32_t Http2::write_time_threshold            = 100;
-uint32_t Http2::buffer_water_mark               = 0;
+float Http2::stream_error_rate_threshold         = 0.1;
+uint32_t Http2::stream_error_sampling_threshold  = 10;
+uint32_t Http2::max_settings_per_frame           = 7;
+uint32_t Http2::max_settings_per_minute          = 14;
+uint32_t Http2::max_settings_frames_per_minute   = 14;
+uint32_t Http2::max_ping_frames_per_minute       = 60;
+uint32_t Http2::max_priority_frames_per_minute   = 120;
+uint32_t Http2::max_rst_stream_frames_per_minute = 200;
+float Http2::min_avg_window_update               = 2560.0;
+uint32_t Http2::con_slow_log_threshold           = 0;
+uint32_t Http2::stream_slow_log_threshold        = 0;
+uint32_t Http2::header_table_size_limit          = 65536;
+uint32_t Http2::write_buffer_block_size          = 262144;
+float Http2::write_size_threshold                = 0.5;
+uint32_t Http2::write_time_threshold             = 100;
+uint32_t Http2::buffer_water_mark                = 0;
 
 void
 Http2::init()
@@ -541,6 +542,7 @@ Http2::init()
   REC_EstablishStaticConfigInt32U(max_settings_frames_per_minute, "proxy.config.http2.max_settings_frames_per_minute");
   REC_EstablishStaticConfigInt32U(max_ping_frames_per_minute, "proxy.config.http2.max_ping_frames_per_minute");
   REC_EstablishStaticConfigInt32U(max_priority_frames_per_minute, "proxy.config.http2.max_priority_frames_per_minute");
+  REC_EstablishStaticConfigInt32U(max_rst_stream_frames_per_minute, "proxy.config.http2.max_rst_stream_frames_per_minute");
   REC_EstablishStaticConfigFloat(min_avg_window_update, "proxy.config.http2.min_avg_window_update");
   REC_EstablishStaticConfigInt32U(con_slow_log_threshold, "proxy.config.http2.connection.slow.log.threshold");
   REC_EstablishStaticConfigInt32U(stream_slow_log_threshold, "proxy.config.http2.stream.slow.log.threshold");
@@ -596,6 +598,8 @@ Http2::init()
   http2_rsb.max_ping_frames_per_minute_exceeded = intm.newMetricPtr("proxy.process.http2.max_ping_frames_per_minute_exceeded");
   http2_rsb.max_priority_frames_per_minute_exceeded =
     intm.newMetricPtr("proxy.process.http2.max_priority_frames_per_minute_exceeded");
+  http2_rsb.max_rst_stream_frames_per_minute_exceeded =
+    intm.newMetricPtr("proxy.process.http2.max_rst_stream_frames_per_minute_exceeded");
   http2_rsb.insufficient_avg_window_update      = intm.newMetricPtr("proxy.process.http2.insufficient_avg_window_update");
   http2_rsb.max_concurrent_streams_exceeded_in  = intm.newMetricPtr("proxy.process.http2.max_concurrent_streams_exceeded_in");
   http2_rsb.max_concurrent_streams_exceeded_out = intm.newMetricPtr("proxy.process.http2.max_concurrent_streams_exceeded_out");

proxy/http2/HTTP2.h

@@ -102,6 +102,7 @@ struct Http2StatsBlock {
   Metrics::IntType *max_settings_frames_per_minute_exceeded;
   Metrics::IntType *max_ping_frames_per_minute_exceeded;
   Metrics::IntType *max_priority_frames_per_minute_exceeded;
+  Metrics::IntType *max_rst_stream_frames_per_minute_exceeded;
   Metrics::IntType *insufficient_avg_window_update;
   Metrics::IntType *max_concurrent_streams_exceeded_in;
   Metrics::IntType *max_concurrent_streams_exceeded_out;
@@ -412,6 +413,7 @@ class Http2
   static uint32_t max_settings_frames_per_minute;
   static uint32_t max_ping_frames_per_minute;
   static uint32_t max_priority_frames_per_minute;
+  static uint32_t max_rst_stream_frames_per_minute;
   static float min_avg_window_update;
   static uint32_t con_slow_log_threshold;
   static uint32_t stream_slow_log_threshold;

proxy/http2/Http2ConnectionState.cc

@@ -551,8 +551,8 @@ Http2ConnectionState::rcv_priority_frame(const Http2Frame &frame)
   // Update PRIORITY frame count per minute
   this->increment_received_priority_frame_count();
   // Close this connection if its priority frame count received exceeds a limit
-  if (Http2::max_priority_frames_per_minute != 0 &&
-      this->get_received_priority_frame_count() > Http2::max_priority_frames_per_minute) {
+  if (configured_max_priority_frames_per_minute != 0 &&
+      this->get_received_priority_frame_count() > configured_max_priority_frames_per_minute) {
     Metrics::increment(http2_rsb.max_priority_frames_per_minute_exceeded);
     Http2StreamDebug(this->session, stream_id, "Observed too frequent priority changes: %u priority changes within a last minute",
                      this->get_received_priority_frame_count());
@@ -623,6 +623,18 @@ Http2ConnectionState::rcv_rst_stream_frame(const Http2Frame &frame)
                       "reset frame wrong length");
   }
 
+  // Update RST_STREAM frame count per minute
+  this->increment_received_rst_stream_frame_count();
+  // Close this connection if its RST_STREAM frame count exceeds a limit
+  if (configured_max_rst_stream_frames_per_minute != 0 &&
+      this->get_received_rst_stream_frame_count() > configured_max_rst_stream_frames_per_minute) {
+    Metrics::increment(http2_rsb.max_rst_stream_frames_per_minute_exceeded);
+    Http2StreamDebug(this->session, stream_id, "Observed too frequent RST_STREAM frames: %u frames within a last minute",
+                     this->get_received_settings_frame_count());
+    return Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_ENHANCE_YOUR_CALM,
+                      "reset too frequent RST_STREAM frames");
+  }
+
   if (stream == nullptr || !stream->change_state(frame.header().type, frame.header().flags)) {
     // If a RST_STREAM frame identifying an idle stream is received, the
     // recipient MUST treat this as a connection error of type PROTOCOL_ERROR.
@@ -663,7 +675,8 @@ Http2ConnectionState::rcv_settings_frame(const Http2Frame &frame)
   // Update SETTINGS frame count per minute
   this->increment_received_settings_frame_count();
   // Close this connection if its SETTINGS frame count exceeds a limit
-  if (this->get_received_settings_frame_count() > Http2::max_settings_frames_per_minute) {
+  if (configured_max_settings_frames_per_minute != 0 &&
+      this->get_received_settings_frame_count() > configured_max_settings_frames_per_minute) {
     Metrics::increment(http2_rsb.max_settings_frames_per_minute_exceeded);
     Http2StreamDebug(this->session, stream_id, "Observed too frequent SETTINGS frames: %u frames within a last minute",
                      this->get_received_settings_frame_count());
@@ -798,7 +811,7 @@ Http2ConnectionState::rcv_ping_frame(const Http2Frame &frame)
   // Update PING frame count per minute
   this->increment_received_ping_frame_count();
   // Close this connection if its ping count received exceeds a limit
-  if (this->get_received_ping_frame_count() > Http2::max_ping_frames_per_minute) {
+  if (configured_max_ping_frames_per_minute != 0 && this->get_received_ping_frame_count() > configured_max_ping_frames_per_minute) {
     Metrics::increment(http2_rsb.max_ping_frames_per_minute_exceeded);
     Http2StreamDebug(this->session, stream_id, "Observed too frequent PING frames: %u PING frames within a last minute",
                      this->get_received_ping_frame_count());
@@ -1220,6 +1233,25 @@ Http2ConnectionState::init(Http2CommonSession *ssn)
     dependency_tree = new DependencyTree(this->_get_configured_max_concurrent_streams());
   }
 
+  configured_max_settings_frames_per_minute   = Http2::max_settings_frames_per_minute;
+  configured_max_ping_frames_per_minute       = Http2::max_ping_frames_per_minute;
+  configured_max_priority_frames_per_minute   = Http2::max_priority_frames_per_minute;
+  configured_max_rst_stream_frames_per_minute = Http2::max_rst_stream_frames_per_minute;
+  if (auto snis = session->get_netvc()->get_service<TLSSNISupport>(); snis) {
+    if (snis->hints_from_sni.http2_max_settings_frames_per_minute.has_value()) {
+      configured_max_settings_frames_per_minute = snis->hints_from_sni.http2_max_settings_frames_per_minute.value();
+    }
+    if (snis->hints_from_sni.http2_max_ping_frames_per_minute.has_value()) {
+      configured_max_ping_frames_per_minute = snis->hints_from_sni.http2_max_ping_frames_per_minute.value();
+    }
+    if (snis->hints_from_sni.http2_max_priority_frames_per_minute.has_value()) {
+      configured_max_priority_frames_per_minute = snis->hints_from_sni.http2_max_priority_frames_per_minute.value();
+    }
+    if (snis->hints_from_sni.http2_max_rst_stream_frames_per_minute.has_value()) {
+      configured_max_rst_stream_frames_per_minute = snis->hints_from_sni.http2_max_rst_stream_frames_per_minute.value();
+    }
+  }
+
   _cop = ActivityCop<Http2Stream>(this->mutex, &stream_list, 1);
   _cop.start();
 }
@@ -2576,6 +2608,18 @@ Http2ConnectionState::get_received_priority_frame_count()
   return this->_received_priority_frame_counter.get_count();
 }
 
+void
+Http2ConnectionState::increment_received_rst_stream_frame_count()
+{
+  this->_received_rst_stream_frame_counter.increment();
+}
+
+uint32_t
+Http2ConnectionState::get_received_rst_stream_frame_count()
+{
+  return this->_received_rst_stream_frame_counter.get_count();
+}
+
 // Return min_concurrent_streams_in when current client streams number is larger than max_active_streams_in.
 // Main purpose of this is preventing DDoS Attacks.
 unsigned

proxy/http2/Http2ConnectionState.h

@@ -194,6 +194,8 @@ class Http2ConnectionState : public Continuation
   uint32_t get_received_ping_frame_count();
   void increment_received_priority_frame_count();
   uint32_t get_received_priority_frame_count();
+  void increment_received_rst_stream_frame_count();
+  uint32_t get_received_rst_stream_frame_count();
 
   ssize_t get_peer_rwnd() const;
   Http2ErrorCode increment_peer_rwnd(size_t amount);
@@ -327,6 +329,7 @@ class Http2ConnectionState : public Continuation
   Http2FrequencyCounter _received_settings_frame_counter;
   Http2FrequencyCounter _received_ping_frame_counter;
   Http2FrequencyCounter _received_priority_frame_counter;
+  Http2FrequencyCounter _received_rst_stream_frame_counter;
 
   /** Records the various settings for each SETTINGS frame that we've sent.
    *
@@ -393,6 +396,11 @@ class Http2ConnectionState : public Continuation
   Event *shutdown_cont_event        = nullptr;
   Event *fini_event                 = nullptr;
   Event *zombie_event               = nullptr;
+
+  uint32_t configured_max_settings_frames_per_minute   = 0;
+  uint32_t configured_max_ping_frames_per_minute       = 0;
+  uint32_t configured_max_priority_frames_per_minute   = 0;
+  uint32_t configured_max_rst_stream_frames_per_minute = 0;
 };
 
 ///////////////////////////////////////////////

src/records/RecordsConfig.cc

@@ -1315,6 +1315,8 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.http2.max_priority_frames_per_minute", RECD_INT, "120", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.http2.max_rst_stream_frames_per_minute", RECD_INT, "200", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
+  ,
   {RECT_CONFIG, "proxy.config.http2.min_avg_window_update", RECD_FLOAT, "2560.0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.http2.header_table_size_limit", RECD_INT, "65536", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}