From 1ab9ce3ba72b15ee3fa9cd6c44114b906feee246 Mon Sep 17 00:00:00 2001
From: Masakazu Kitajo <maskit@apache.org>
Date: Mon, 19 Oct 2020 18:23:25 +0900
Subject: [PATCH 1/8] Reduce the number of write operation on H2

---
 proxy/http2/Http2ClientSession.cc   | 29 ++++++++++++++++++++++++-----
 proxy/http2/Http2ClientSession.h    |  6 +++++-
 proxy/http2/Http2ConnectionState.cc |  2 +-
 3 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc
index fb2650ce757..870f8acee0f 100644
--- a/proxy/http2/Http2ClientSession.cc
+++ b/proxy/http2/Http2ClientSession.cc
@@ -163,6 +163,7 @@ Http2ClientSession::free()
 
   free_MIOBuffer(this->read_buffer);
   free_MIOBuffer(this->write_buffer);
+  free_MIOBuffer(this->_send_frame_buffer);
   THREAD_FREE(this, http2ClientSessionAllocator, this_ethread());
 }
 
@@ -177,6 +178,9 @@ Http2ClientSession::start()
   VIO *read_vio = this->do_io_read(this, INT64_MAX, this->read_buffer);
   write_vio     = this->do_io_write(this, INT64_MAX, this->sm_writer);
 
+  this->_send_frame_buffer        = new_MIOBuffer(BUFFER_SIZE_INDEX_1M);
+  this->_send_frame_buffer_reader = this->_send_frame_buffer->alloc_reader();
+
   this->connection_state.init();
   send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_INIT, this);
 
@@ -299,13 +303,28 @@ Http2ClientSession::set_half_close_local_flag(bool flag)
 }
 
 int64_t
-Http2ClientSession::xmit(const Http2TxFrame &frame)
+Http2ClientSession::xmit(const Http2TxFrame &frame, bool flush)
 {
-  int64_t len = frame.write_to(this->write_buffer);
+  int64_t len = 0;
+
+  this->_send_frame_buffer_used_size += frame.write_to(this->_send_frame_buffer);
 
-  if (len > 0) {
-    total_write_len += len;
-    write_reenable();
+  if (!flush) {
+    if (this->_send_frame_buffer_used_size >= (512 * 1024)) {
+      flush = true;
+    }
+  }
+
+  if (flush) {
+    len = this->write_buffer->write(this->_send_frame_buffer_reader);
+    this->_send_frame_buffer_reader->consume(len);
+    _send_frame_buffer_used_size -= len;
+    ink_release_assert(_send_frame_buffer_used_size == 0);
+
+    if (len > 0) {
+      total_write_len += len;
+      write_reenable();
+    }
   }
 
   return len;
diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h
index 0f1f64bb1cc..aac3d172332 100644
--- a/proxy/http2/Http2ClientSession.h
+++ b/proxy/http2/Http2ClientSession.h
@@ -101,7 +101,7 @@ class Http2ClientSession : public ProxySession
 
   // more methods
   void write_reenable();
-  int64_t xmit(const Http2TxFrame &frame);
+  int64_t xmit(const Http2TxFrame &frame, bool flush = true);
 
   ////////////////////
   // Accessors
@@ -183,6 +183,10 @@ class Http2ClientSession : public ProxySession
   Event *_reenable_event = nullptr;
   int _n_frame_read      = 0;
 
+  MIOBuffer *_send_frame_buffer             = nullptr;
+  IOBufferReader *_send_frame_buffer_reader = nullptr;
+  uint32_t _send_frame_buffer_used_size     = 0;
+
   int64_t read_from_early_data   = 0;
   bool cur_frame_from_early_data = false;
 };
diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc
index 1f57a881d05..87caaad350e 100644
--- a/proxy/http2/Http2ConnectionState.cc
+++ b/proxy/http2/Http2ConnectionState.cc
@@ -1580,7 +1580,7 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len
                    _client_rwnd, stream->client_rwnd(), payload_length);
 
   Http2DataFrame data(stream->get_id(), flags, resp_reader, payload_length);
-  this->ua_session->xmit(data);
+  this->ua_session->xmit(data, flags & HTTP2_FLAGS_DATA_END_STREAM);
 
   stream->update_sent_count(payload_length);
 

From 7e2ce1dffae51cad2c5c01151a54c75de58fedaf Mon Sep 17 00:00:00 2001
From: Masakazu Kitajo <maskit@apache.org>
Date: Tue, 20 Oct 2020 11:51:49 +0900
Subject: [PATCH 2/8] Reuse write_buffer instead of the new buffer and increase
 the block size

---
 proxy/http2/Http2ClientSession.cc | 29 ++++++++++++-----------------
 proxy/http2/Http2ClientSession.h  |  4 +---
 2 files changed, 13 insertions(+), 20 deletions(-)

diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc
index 870f8acee0f..2201272f520 100644
--- a/proxy/http2/Http2ClientSession.cc
+++ b/proxy/http2/Http2ClientSession.cc
@@ -163,7 +163,6 @@ Http2ClientSession::free()
 
   free_MIOBuffer(this->read_buffer);
   free_MIOBuffer(this->write_buffer);
-  free_MIOBuffer(this->_send_frame_buffer);
   THREAD_FREE(this, http2ClientSessionAllocator, this_ethread());
 }
 
@@ -178,9 +177,6 @@ Http2ClientSession::start()
   VIO *read_vio = this->do_io_read(this, INT64_MAX, this->read_buffer);
   write_vio     = this->do_io_write(this, INT64_MAX, this->sm_writer);
 
-  this->_send_frame_buffer        = new_MIOBuffer(BUFFER_SIZE_INDEX_1M);
-  this->_send_frame_buffer_reader = this->_send_frame_buffer->alloc_reader();
-
   this->connection_state.init();
   send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_INIT, this);
 
@@ -221,8 +217,8 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB
   this->read_buffer->water_mark = connection_state.server_settings.get(HTTP2_SETTINGS_MAX_FRAME_SIZE);
   this->_reader                 = reader ? reader : this->read_buffer->alloc_reader();
 
-  // Set write buffer size to max size of TLS record (16KB)
-  this->write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_16K);
+  // This block size is the buffer size that we pass to SSLWriteBuffer
+  this->write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_1M);
   this->sm_writer    = this->write_buffer->alloc_reader();
 
   this->_handle_if_ssl(new_vc);
@@ -305,24 +301,23 @@ Http2ClientSession::set_half_close_local_flag(bool flag)
 int64_t
 Http2ClientSession::xmit(const Http2TxFrame &frame, bool flush)
 {
-  int64_t len = 0;
-
-  this->_send_frame_buffer_used_size += frame.write_to(this->_send_frame_buffer);
+  int64_t len = frame.write_to(this->write_buffer);
+  this->_pending_sending_data_size += len;
 
+  // Force flush for some cases
   if (!flush) {
-    if (this->_send_frame_buffer_used_size >= (512 * 1024)) {
+    // Flush if we already use half of the buffer to avoid adding a new block to the chain.
+    // A frame size can be 16MB at maximum so blocks can be added, but that's fine.
+    if (this->_pending_sending_data_size >= (512 * 1024)) {
       flush = true;
     }
+    // We may want to flush if the last flush was over 200 ms ago.
   }
 
   if (flush) {
-    len = this->write_buffer->write(this->_send_frame_buffer_reader);
-    this->_send_frame_buffer_reader->consume(len);
-    _send_frame_buffer_used_size -= len;
-    ink_release_assert(_send_frame_buffer_used_size == 0);
-
-    if (len > 0) {
-      total_write_len += len;
+    if (this->_pending_sending_data_size > 0) {
+      total_write_len += this->_pending_sending_data_size;
+      this->_pending_sending_data_size = 0;
       write_reenable();
     }
   }
diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h
index aac3d172332..c214acc21f4 100644
--- a/proxy/http2/Http2ClientSession.h
+++ b/proxy/http2/Http2ClientSession.h
@@ -183,9 +183,7 @@ class Http2ClientSession : public ProxySession
   Event *_reenable_event = nullptr;
   int _n_frame_read      = 0;
 
-  MIOBuffer *_send_frame_buffer             = nullptr;
-  IOBufferReader *_send_frame_buffer_reader = nullptr;
-  uint32_t _send_frame_buffer_used_size     = 0;
+  uint32_t _pending_sending_data_size = 0;
 
   int64_t read_from_early_data   = 0;
   bool cur_frame_from_early_data = false;

From bccfe896b8fe9ad7ba9b71484f8e4351ca1b8cb8 Mon Sep 17 00:00:00 2001
From: Masakazu Kitajo <maskit@apache.org>
Date: Tue, 20 Oct 2020 15:50:10 +0900
Subject: [PATCH 3/8] Flush write buffer if a session becomes idle

---
 proxy/http2/Http2ClientSession.cc   | 16 +++++++++++-----
 proxy/http2/Http2ClientSession.h    |  1 +
 proxy/http2/Http2ConnectionState.cc |  2 ++
 3 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc
index 2201272f520..32e847d4a12 100644
--- a/proxy/http2/Http2ClientSession.cc
+++ b/proxy/http2/Http2ClientSession.cc
@@ -315,16 +315,22 @@ Http2ClientSession::xmit(const Http2TxFrame &frame, bool flush)
   }
 
   if (flush) {
-    if (this->_pending_sending_data_size > 0) {
-      total_write_len += this->_pending_sending_data_size;
-      this->_pending_sending_data_size = 0;
-      write_reenable();
-    }
+    this->flush();
   }
 
   return len;
 }
 
+void
+Http2ClientSession::flush()
+{
+  if (this->_pending_sending_data_size > 0) {
+    total_write_len += this->_pending_sending_data_size;
+    this->_pending_sending_data_size = 0;
+    write_reenable();
+  }
+}
+
 int
 Http2ClientSession::main_event_handler(int event, void *edata)
 {
diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h
index c214acc21f4..f09cd6d0d9f 100644
--- a/proxy/http2/Http2ClientSession.h
+++ b/proxy/http2/Http2ClientSession.h
@@ -102,6 +102,7 @@ class Http2ClientSession : public ProxySession
   // more methods
   void write_reenable();
   int64_t xmit(const Http2TxFrame &frame, bool flush = true);
+  void flush();
 
   ////////////////////
   // Accessors
diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc
index 87caaad350e..b2597e80923 100644
--- a/proxy/http2/Http2ConnectionState.cc
+++ b/proxy/http2/Http2ConnectionState.cc
@@ -1547,6 +1547,7 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len
     // We only need to check for window size when there is a payload
     if (window_size <= 0) {
       Http2StreamDebug(this->ua_session, stream->get_id(), "No window");
+      this->ua_session->flush();
       return Http2SendDataFrameResult::NO_WINDOW;
     }
 
@@ -1564,6 +1565,7 @@ Http2ConnectionState::send_a_data_frame(Http2Stream *stream, size_t &payload_len
   // OK if there is no body yet. Otherwise continue on to send a DATA frame and delete the stream
   if (!stream->is_write_vio_done() && payload_length == 0) {
     Http2StreamDebug(this->ua_session, stream->get_id(), "No payload");
+    this->ua_session->flush();
     return Http2SendDataFrameResult::NO_PAYLOAD;
   }
 

From 9bf83e40a71498dbfc4302e3190e5cd20bc7e1de Mon Sep 17 00:00:00 2001
From: Masakazu Kitajo <maskit@apache.org>
Date: Thu, 22 Oct 2020 17:06:58 +0900
Subject: [PATCH 4/8] Make the numbers configurable

---
 doc/admin-guide/files/records.config.en.rst | 13 +++++++++++++
 mgmt/RecordsConfig.cc                       |  4 ++++
 proxy/http2/HTTP2.cc                        |  4 ++++
 proxy/http2/HTTP2.h                         |  2 ++
 proxy/http2/Http2ClientSession.cc           |  7 ++++---
 proxy/http2/Http2ClientSession.h            |  1 +
 6 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst
index 7e4a97e5bce..9f6ea92211a 100644
--- a/doc/admin-guide/files/records.config.en.rst
+++ b/doc/admin-guide/files/records.config.en.rst
@@ -3901,6 +3901,19 @@ HTTP/2 Configuration
    Clients that send smaller window increments lower than this limit will be immediately disconnected with an error
    code of ENHANCE_YOUR_CALM.
 
+.. ts:cv:: CONFIG proxy.config.http2.write_buffer_block_size_index INT 11
+   :reloadable:
+
+   Specifies the size of a buffer block that is used for buffering outgoing
+   HTTP/2 frames. The default value is 11 which maps to 256KB.
+
+.. ts:cv:: CONFIG proxy.config.http2.write_threshold FLOAT `0.5
+   :reloadable:
+
+   Specifies the threshold for triggering write operation for sending HTTP/2
+   frames. The default value is 0.5 and it measn write operation is going to be
+   triggered when half or more of the buffer is occupied.
+
 HTTP/3 Configuration
 ====================
 
diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc
index 30ecbcc8dff..120472fcb56 100644
--- a/mgmt/RecordsConfig.cc
+++ b/mgmt/RecordsConfig.cc
@@ -1333,6 +1333,10 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.http2.header_table_size_limit", RECD_INT, "65536", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.http2.write_buffer_block_size_index", RECD_INT, "11", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
+  {RECT_CONFIG, "proxy.config.http2.write_threshold", RECD_FLOAT, "0.5",  RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
 
   //############
   //#
diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc
index 4254dd48dc3..a3aa62c1437 100644
--- a/proxy/http2/HTTP2.cc
+++ b/proxy/http2/HTTP2.cc
@@ -805,6 +805,8 @@ float Http2::min_avg_window_update             = 2560.0;
 uint32_t Http2::con_slow_log_threshold         = 0;
 uint32_t Http2::stream_slow_log_threshold      = 0;
 uint32_t Http2::header_table_size_limit        = 65536;
+uint32_t Http2::write_buffer_block_size_index  = 11;
+float Http2::write_threshold                   = 0.5;
 
 void
 Http2::init()
@@ -832,6 +834,8 @@ Http2::init()
   REC_EstablishStaticConfigInt32U(con_slow_log_threshold, "proxy.config.http2.connection.slow.log.threshold");
   REC_EstablishStaticConfigInt32U(stream_slow_log_threshold, "proxy.config.http2.stream.slow.log.threshold");
   REC_EstablishStaticConfigInt32U(header_table_size_limit, "proxy.config.http2.header_table_size_limit");
+  REC_EstablishStaticConfigInt32U(write_buffer_block_size_index, "proxy.config.http2.write_buffer_block_size_index");
+  REC_EstablishStaticConfigFloat(write_threshold, "proxy.config.http2.write_threshold");
 
   // If any settings is broken, ATS should not start
   ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, max_concurrent_streams_in}));
diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h
index 966164508a5..7c09b544b48 100644
--- a/proxy/http2/HTTP2.h
+++ b/proxy/http2/HTTP2.h
@@ -403,6 +403,8 @@ class Http2
   static uint32_t con_slow_log_threshold;
   static uint32_t stream_slow_log_threshold;
   static uint32_t header_table_size_limit;
+  static uint32_t write_buffer_block_size_index;
+  static float write_threshold;
 
   static void init();
 };
diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc
index 32e847d4a12..1c5d3f4940c 100644
--- a/proxy/http2/Http2ClientSession.cc
+++ b/proxy/http2/Http2ClientSession.cc
@@ -218,8 +218,9 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB
   this->_reader                 = reader ? reader : this->read_buffer->alloc_reader();
 
   // This block size is the buffer size that we pass to SSLWriteBuffer
-  this->write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_1M);
-  this->sm_writer    = this->write_buffer->alloc_reader();
+  this->write_buffer     = new_MIOBuffer(Http2::write_buffer_block_size_index);
+  this->sm_writer        = this->write_buffer->alloc_reader();
+  this->_write_threshold = index_to_buffer_size(Http2::write_buffer_block_size_index) * Http2::write_threshold;
 
   this->_handle_if_ssl(new_vc);
 
@@ -308,7 +309,7 @@ Http2ClientSession::xmit(const Http2TxFrame &frame, bool flush)
   if (!flush) {
     // Flush if we already use half of the buffer to avoid adding a new block to the chain.
     // A frame size can be 16MB at maximum so blocks can be added, but that's fine.
-    if (this->_pending_sending_data_size >= (512 * 1024)) {
+    if (this->_pending_sending_data_size >= this->_write_threshold) {
       flush = true;
     }
     // We may want to flush if the last flush was over 200 ms ago.
diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h
index f09cd6d0d9f..b5e13077efc 100644
--- a/proxy/http2/Http2ClientSession.h
+++ b/proxy/http2/Http2ClientSession.h
@@ -162,6 +162,7 @@ class Http2ClientSession : public ProxySession
   MIOBuffer *write_buffer        = nullptr;
   IOBufferReader *sm_writer      = nullptr;
   Http2FrameHeader current_hdr   = {0, 0, 0, 0};
+  uint32_t _write_threshold      = 0;
 
   IpEndpoint cached_client_addr;
   IpEndpoint cached_local_addr;

From f59bf8da93495c902b060c5ad5eb441b9e858ede Mon Sep 17 00:00:00 2001
From: Masakazu Kitajo <maskit@apache.org>
Date: Mon, 26 Oct 2020 07:54:28 +0900
Subject: [PATCH 5/8] Change the way to specify buffer block size to actual
 buffer size

---
 doc/admin-guide/files/records.config.en.rst | 4 ++--
 mgmt/RecordsConfig.cc                       | 2 +-
 proxy/http2/HTTP2.cc                        | 4 ++--
 proxy/http2/HTTP2.h                         | 2 +-
 proxy/http2/Http2ClientSession.cc           | 7 ++++---
 5 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst
index 9f6ea92211a..488297dbe7a 100644
--- a/doc/admin-guide/files/records.config.en.rst
+++ b/doc/admin-guide/files/records.config.en.rst
@@ -3901,11 +3901,11 @@ HTTP/2 Configuration
    Clients that send smaller window increments lower than this limit will be immediately disconnected with an error
    code of ENHANCE_YOUR_CALM.
 
-.. ts:cv:: CONFIG proxy.config.http2.write_buffer_block_size_index INT 11
+.. ts:cv:: CONFIG proxy.config.http2.write_buffer_block_size INT 262144
    :reloadable:
 
    Specifies the size of a buffer block that is used for buffering outgoing
-   HTTP/2 frames. The default value is 11 which maps to 256KB.
+   HTTP/2 frames. The size will be rounded up based on power of 2.
 
 .. ts:cv:: CONFIG proxy.config.http2.write_threshold FLOAT `0.5
    :reloadable:
diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc
index 120472fcb56..9a3e32437d1 100644
--- a/mgmt/RecordsConfig.cc
+++ b/mgmt/RecordsConfig.cc
@@ -1333,7 +1333,7 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.http2.header_table_size_limit", RECD_INT, "65536", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
   ,
-  {RECT_CONFIG, "proxy.config.http2.write_buffer_block_size_index", RECD_INT, "11", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  {RECT_CONFIG, "proxy.config.http2.write_buffer_block_size", RECD_INT, "262144", RECU_DYNAMIC, RR_NULL, RECC_NULL, "^[0-9]+$", RECA_NULL}
   ,
   {RECT_CONFIG, "proxy.config.http2.write_threshold", RECD_FLOAT, "0.5",  RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc
index a3aa62c1437..a1dfa4dd346 100644
--- a/proxy/http2/HTTP2.cc
+++ b/proxy/http2/HTTP2.cc
@@ -805,7 +805,7 @@ float Http2::min_avg_window_update             = 2560.0;
 uint32_t Http2::con_slow_log_threshold         = 0;
 uint32_t Http2::stream_slow_log_threshold      = 0;
 uint32_t Http2::header_table_size_limit        = 65536;
-uint32_t Http2::write_buffer_block_size_index  = 11;
+uint32_t Http2::write_buffer_block_size        = 262144;
 float Http2::write_threshold                   = 0.5;
 
 void
@@ -834,7 +834,7 @@ Http2::init()
   REC_EstablishStaticConfigInt32U(con_slow_log_threshold, "proxy.config.http2.connection.slow.log.threshold");
   REC_EstablishStaticConfigInt32U(stream_slow_log_threshold, "proxy.config.http2.stream.slow.log.threshold");
   REC_EstablishStaticConfigInt32U(header_table_size_limit, "proxy.config.http2.header_table_size_limit");
-  REC_EstablishStaticConfigInt32U(write_buffer_block_size_index, "proxy.config.http2.write_buffer_block_size_index");
+  REC_EstablishStaticConfigInt32U(write_buffer_block_size, "proxy.config.http2.write_buffer_block_size");
   REC_EstablishStaticConfigFloat(write_threshold, "proxy.config.http2.write_threshold");
 
   // If any settings is broken, ATS should not start
diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h
index 7c09b544b48..e4486f17d12 100644
--- a/proxy/http2/HTTP2.h
+++ b/proxy/http2/HTTP2.h
@@ -403,7 +403,7 @@ class Http2
   static uint32_t con_slow_log_threshold;
   static uint32_t stream_slow_log_threshold;
   static uint32_t header_table_size_limit;
-  static uint32_t write_buffer_block_size_index;
+  static uint32_t write_buffer_block_size;
   static float write_threshold;
 
   static void init();
diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc
index 1c5d3f4940c..b3f89583826 100644
--- a/proxy/http2/Http2ClientSession.cc
+++ b/proxy/http2/Http2ClientSession.cc
@@ -218,9 +218,10 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB
   this->_reader                 = reader ? reader : this->read_buffer->alloc_reader();
 
   // This block size is the buffer size that we pass to SSLWriteBuffer
-  this->write_buffer     = new_MIOBuffer(Http2::write_buffer_block_size_index);
-  this->sm_writer        = this->write_buffer->alloc_reader();
-  this->_write_threshold = index_to_buffer_size(Http2::write_buffer_block_size_index) * Http2::write_threshold;
+  auto buffer_block_size_index = iobuffer_size_to_index(Http2::write_buffer_block_size, MAX_BUFFER_SIZE_INDEX);
+  this->write_buffer           = new_MIOBuffer(buffer_block_size_index);
+  this->sm_writer              = this->write_buffer->alloc_reader();
+  this->_write_threshold       = index_to_buffer_size(buffer_block_size_index) * Http2::write_threshold;
 
   this->_handle_if_ssl(new_vc);
 

From c5ad607e767f33362b77076b9b5d6d1f111601c2 Mon Sep 17 00:00:00 2001
From: Masakazu Kitajo <maskit@apache.org>
Date: Mon, 26 Oct 2020 10:17:38 +0900
Subject: [PATCH 6/8] Flush at least every X ms

---
 mgmt/RecordsConfig.cc             |  6 ++++--
 proxy/http2/HTTP2.cc              |  6 ++++--
 proxy/http2/HTTP2.h               |  3 ++-
 proxy/http2/Http2ClientSession.cc | 11 ++++++-----
 proxy/http2/Http2ClientSession.h  | 18 ++++++++++--------
 5 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc
index 9a3e32437d1..41a98e3cc05 100644
--- a/mgmt/RecordsConfig.cc
+++ b/mgmt/RecordsConfig.cc
@@ -1333,9 +1333,11 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.http2.header_table_size_limit", RECD_INT, "65536", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
   ,
-  {RECT_CONFIG, "proxy.config.http2.write_buffer_block_size", RECD_INT, "262144", RECU_DYNAMIC, RR_NULL, RECC_NULL, "^[0-9]+$", RECA_NULL}
+  {RECT_CONFIG, "proxy.config.http2.write_buffer_block_size", RECD_INT, "262144", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
   ,
-  {RECT_CONFIG, "proxy.config.http2.write_threshold", RECD_FLOAT, "0.5",  RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  {RECT_CONFIG, "proxy.config.http2.write_size_threshold", RECD_FLOAT, "0.5",  RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
+  {RECT_CONFIG, "proxy.config.http2.write_time_threshold", RECD_INT, "100", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
   ,
 
   //############
diff --git a/proxy/http2/HTTP2.cc b/proxy/http2/HTTP2.cc
index a1dfa4dd346..50a5d236440 100644
--- a/proxy/http2/HTTP2.cc
+++ b/proxy/http2/HTTP2.cc
@@ -806,7 +806,8 @@ uint32_t Http2::con_slow_log_threshold         = 0;
 uint32_t Http2::stream_slow_log_threshold      = 0;
 uint32_t Http2::header_table_size_limit        = 65536;
 uint32_t Http2::write_buffer_block_size        = 262144;
-float Http2::write_threshold                   = 0.5;
+float Http2::write_size_threshold              = 0.5;
+uint32_t Http2::write_time_threshold           = 100;
 
 void
 Http2::init()
@@ -835,7 +836,8 @@ Http2::init()
   REC_EstablishStaticConfigInt32U(stream_slow_log_threshold, "proxy.config.http2.stream.slow.log.threshold");
   REC_EstablishStaticConfigInt32U(header_table_size_limit, "proxy.config.http2.header_table_size_limit");
   REC_EstablishStaticConfigInt32U(write_buffer_block_size, "proxy.config.http2.write_buffer_block_size");
-  REC_EstablishStaticConfigFloat(write_threshold, "proxy.config.http2.write_threshold");
+  REC_EstablishStaticConfigFloat(write_size_threshold, "proxy.config.http2.write_size_threshold");
+  REC_EstablishStaticConfigInt32U(write_time_threshold, "proxy.config.http2.write_time_threshold");
 
   // If any settings is broken, ATS should not start
   ink_release_assert(http2_settings_parameter_is_valid({HTTP2_SETTINGS_MAX_CONCURRENT_STREAMS, max_concurrent_streams_in}));
diff --git a/proxy/http2/HTTP2.h b/proxy/http2/HTTP2.h
index e4486f17d12..d2eed22ffe3 100644
--- a/proxy/http2/HTTP2.h
+++ b/proxy/http2/HTTP2.h
@@ -404,7 +404,8 @@ class Http2
   static uint32_t stream_slow_log_threshold;
   static uint32_t header_table_size_limit;
   static uint32_t write_buffer_block_size;
-  static float write_threshold;
+  static float write_size_threshold;
+  static uint32_t write_time_threshold;
 
   static void init();
 };
diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc
index b3f89583826..4bd6edf3c55 100644
--- a/proxy/http2/Http2ClientSession.cc
+++ b/proxy/http2/Http2ClientSession.cc
@@ -221,7 +221,7 @@ Http2ClientSession::new_connection(NetVConnection *new_vc, MIOBuffer *iobuf, IOB
   auto buffer_block_size_index = iobuffer_size_to_index(Http2::write_buffer_block_size, MAX_BUFFER_SIZE_INDEX);
   this->write_buffer           = new_MIOBuffer(buffer_block_size_index);
   this->sm_writer              = this->write_buffer->alloc_reader();
-  this->_write_threshold       = index_to_buffer_size(buffer_block_size_index) * Http2::write_threshold;
+  this->_write_size_threshold  = index_to_buffer_size(buffer_block_size_index) * Http2::write_size_threshold;
 
   this->_handle_if_ssl(new_vc);
 
@@ -305,15 +305,13 @@ Http2ClientSession::xmit(const Http2TxFrame &frame, bool flush)
 {
   int64_t len = frame.write_to(this->write_buffer);
   this->_pending_sending_data_size += len;
-
   // Force flush for some cases
   if (!flush) {
     // Flush if we already use half of the buffer to avoid adding a new block to the chain.
     // A frame size can be 16MB at maximum so blocks can be added, but that's fine.
-    if (this->_pending_sending_data_size >= this->_write_threshold) {
+    if (this->_pending_sending_data_size >= this->_write_size_threshold) {
       flush = true;
     }
-    // We may want to flush if the last flush was over 200 ms ago.
   }
 
   if (flush) {
@@ -329,6 +327,7 @@ Http2ClientSession::flush()
   if (this->_pending_sending_data_size > 0) {
     total_write_len += this->_pending_sending_data_size;
     this->_pending_sending_data_size = 0;
+    this->_write_buffer_last_flash   = Thread::get_hrtime();
     write_reenable();
   }
 }
@@ -377,7 +376,9 @@ Http2ClientSession::main_event_handler(int event, void *edata)
   case VC_EVENT_WRITE_READY:
   case VC_EVENT_WRITE_COMPLETE:
     this->connection_state.restart_streams();
-
+    if ((Thread::get_hrtime() >= this->_write_buffer_last_flash + HRTIME_MSECONDS(this->_write_time_threshold))) {
+      this->flush();
+    }
     retval = 0;
     break;
 
diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h
index b5e13077efc..519d6777429 100644
--- a/proxy/http2/Http2ClientSession.h
+++ b/proxy/http2/Http2ClientSession.h
@@ -155,14 +155,16 @@ class Http2ClientSession : public ProxySession
 
   bool _should_do_something_else();
 
-  int64_t total_write_len        = 0;
-  SessionHandler session_handler = nullptr;
-  MIOBuffer *read_buffer         = nullptr;
-  IOBufferReader *_reader        = nullptr;
-  MIOBuffer *write_buffer        = nullptr;
-  IOBufferReader *sm_writer      = nullptr;
-  Http2FrameHeader current_hdr   = {0, 0, 0, 0};
-  uint32_t _write_threshold      = 0;
+  int64_t total_write_len             = 0;
+  SessionHandler session_handler      = nullptr;
+  MIOBuffer *read_buffer              = nullptr;
+  IOBufferReader *_reader             = nullptr;
+  MIOBuffer *write_buffer             = nullptr;
+  IOBufferReader *sm_writer           = nullptr;
+  Http2FrameHeader current_hdr        = {0, 0, 0, 0};
+  uint32_t _write_size_threshold      = 0;
+  uint32_t _write_time_threshold      = 100;
+  ink_hrtime _write_buffer_last_flash = 0;
 
   IpEndpoint cached_client_addr;
   IpEndpoint cached_local_addr;

From 4ac4569a5bbf08c1fa32d12ea84025aaaa7f0edf Mon Sep 17 00:00:00 2001
From: Masakazu Kitajo <maskit@apache.org>
Date: Mon, 26 Oct 2020 10:35:29 +0900
Subject: [PATCH 7/8] Add documentation for the time threshold

---
 doc/admin-guide/files/records.config.en.rst | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst
index 488297dbe7a..e82055095dd 100644
--- a/doc/admin-guide/files/records.config.en.rst
+++ b/doc/admin-guide/files/records.config.en.rst
@@ -3907,13 +3907,21 @@ HTTP/2 Configuration
    Specifies the size of a buffer block that is used for buffering outgoing
    HTTP/2 frames. The size will be rounded up based on power of 2.
 
-.. ts:cv:: CONFIG proxy.config.http2.write_threshold FLOAT `0.5
+.. ts:cv:: CONFIG proxy.config.http2.write_size_threshold FLOAT 0.5
    :reloadable:
 
-   Specifies the threshold for triggering write operation for sending HTTP/2
+   Specifies the size threshold for triggering write operation for sending HTTP/2
    frames. The default value is 0.5 and it measn write operation is going to be
    triggered when half or more of the buffer is occupied.
 
+.. ts:cv:: CONFIG proxy.config.http2.write_time_threshold INT 100
+   :reloadable:
+   :units: milliseconds
+
+   Specifies the time threshold for triggering write operation for sending HTTP/2
+   frames. Write operation will be triggered at least once every this configured
+   number of millisecond regardless of pending data size.
+
 HTTP/3 Configuration
 ====================
 

From dec942209a37cfddc0af7bb7de93af6c8fbcecad Mon Sep 17 00:00:00 2001
From: Masakazu Kitajo <maskit@apache.org>
Date: Mon, 26 Oct 2020 10:40:49 +0900
Subject: [PATCH 8/8] Fix typo

---
 proxy/http2/Http2ClientSession.cc | 4 ++--
 proxy/http2/Http2ClientSession.h  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/proxy/http2/Http2ClientSession.cc b/proxy/http2/Http2ClientSession.cc
index 4bd6edf3c55..bba7611264f 100644
--- a/proxy/http2/Http2ClientSession.cc
+++ b/proxy/http2/Http2ClientSession.cc
@@ -327,7 +327,7 @@ Http2ClientSession::flush()
   if (this->_pending_sending_data_size > 0) {
     total_write_len += this->_pending_sending_data_size;
     this->_pending_sending_data_size = 0;
-    this->_write_buffer_last_flash   = Thread::get_hrtime();
+    this->_write_buffer_last_flush   = Thread::get_hrtime();
     write_reenable();
   }
 }
@@ -376,7 +376,7 @@ Http2ClientSession::main_event_handler(int event, void *edata)
   case VC_EVENT_WRITE_READY:
   case VC_EVENT_WRITE_COMPLETE:
     this->connection_state.restart_streams();
-    if ((Thread::get_hrtime() >= this->_write_buffer_last_flash + HRTIME_MSECONDS(this->_write_time_threshold))) {
+    if ((Thread::get_hrtime() >= this->_write_buffer_last_flush + HRTIME_MSECONDS(this->_write_time_threshold))) {
       this->flush();
     }
     retval = 0;
diff --git a/proxy/http2/Http2ClientSession.h b/proxy/http2/Http2ClientSession.h
index 519d6777429..c7ba07f2a67 100644
--- a/proxy/http2/Http2ClientSession.h
+++ b/proxy/http2/Http2ClientSession.h
@@ -164,7 +164,7 @@ class Http2ClientSession : public ProxySession
   Http2FrameHeader current_hdr        = {0, 0, 0, 0};
   uint32_t _write_size_threshold      = 0;
   uint32_t _write_time_threshold      = 100;
-  ink_hrtime _write_buffer_last_flash = 0;
+  ink_hrtime _write_buffer_last_flush = 0;
 
   IpEndpoint cached_client_addr;
   IpEndpoint cached_local_addr;