From b81b3a7f5fa7222aece30860d5751c3683a19865 Mon Sep 17 00:00:00 2001 From: Zizhong Zhang Date: Fri, 4 Aug 2017 16:27:48 -0700 Subject: [PATCH 1/2] TS-4042: Add feature to buffer request body before making downstream requests --- example/Makefile.am | 2 + example/request_buffer/request_buffer.c | 148 ++++++++++++++++++ lib/ts/apidefs.h.in | 3 + .../experimental/ts_lua/ts_lua_http_config.c | 2 + proxy/InkAPI.cc | 13 ++ proxy/InkAPITest.cc | 6 +- proxy/api/ts/ts.h | 5 + proxy/http/HttpConfig.cc | 3 + proxy/http/HttpConfig.h | 6 + proxy/http/HttpDebugNames.cc | 6 + proxy/http/HttpSM.cc | 95 ++++++++++- proxy/http/HttpSM.h | 48 ++++++ proxy/http/HttpTransact.cc | 130 ++++++++------- proxy/http/HttpTransact.h | 3 + proxy/http/HttpTunnel.cc | 33 ++-- proxy/http/HttpTunnel.h | 9 +- 16 files changed, 424 insertions(+), 88 deletions(-) create mode 100644 example/request_buffer/request_buffer.c diff --git a/example/Makefile.am b/example/Makefile.am index 27621e3092f..953487a682b 100644 --- a/example/Makefile.am +++ b/example/Makefile.am @@ -32,6 +32,7 @@ example_Plugins = \ blacklist_0.la \ blacklist_1.la \ bnull_transform.la \ + request_buffer.la \ cache_scan.la \ file_1.la \ hello.la \ @@ -98,6 +99,7 @@ basic_auth_la_SOURCES = basic_auth/basic_auth.c blacklist_0_la_SOURCES = blacklist_0/blacklist_0.c blacklist_1_la_SOURCES = blacklist_1/blacklist_1.c bnull_transform_la_SOURCES = bnull_transform/bnull_transform.c +request_buffer_la_SOURCES = request_buffer/request_buffer.c cache_scan_la_SOURCES = cache_scan/cache_scan.cc file_1_la_SOURCES = file_1/file_1.c hello_la_SOURCES = hello/hello.c diff --git a/example/request_buffer/request_buffer.c b/example/request_buffer/request_buffer.c new file mode 100644 index 00000000000..a75650fd067 --- /dev/null +++ b/example/request_buffer/request_buffer.c @@ -0,0 +1,148 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include + +#include "ts/ts.h" +#include "ts/ink_assert.h" +#include "ts/ink_defs.h" + +#define PLUGIN_NAME "request_buffer" + +#define TS_NULL_MUTEX NULL + +static char * +request_body_get(TSHttpTxn txnp, int *len) +{ + char *ret = NULL; + TSIOBufferReader post_buffer_reader = TSHttpTxnPostBufferReaderGet(txnp); + int64_t read_avail = TSIOBufferReaderAvail(post_buffer_reader); + if (read_avail == 0) { + TSIOBufferReaderFree(post_buffer_reader); + return NULL; + } + + ret = (char *)TSmalloc(sizeof(char) * read_avail); + + int64_t consumed = 0; + int64_t data_len = 0; + const char *char_data = NULL; + TSIOBufferBlock block = TSIOBufferReaderStart(post_buffer_reader); + while (block != NULL) { + char_data = TSIOBufferBlockReadStart(block, post_buffer_reader, &data_len); + memcpy(ret + consumed, char_data, data_len); + consumed += data_len; + block = TSIOBufferBlockNext(block); + } + TSIOBufferReaderFree(post_buffer_reader); + + *len = (int)consumed; + return ret; +} + +static int +request_buffer_plugin(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(PLUGIN_NAME, "request_buffer_plugin starting, event[%d]", event); + TSHttpTxn txnp = (TSHttpTxn)(edata); + if (event == TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE) { + int len = 0; + char *body = request_body_get(txnp, &len); + TSDebug(PLUGIN_NAME, "request_buffer_plugin gets the request body with length[%d]", len); + TSfree(body); + TSContDestroy(contp); + } else { + ink_assert(0); + } + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; +} + +bool +is_post_request(TSHttpTxn txnp) +{ + TSMLoc req_loc; + TSMBuffer req_bufp; + if (TSHttpTxnClientReqGet(txnp, &req_bufp, &req_loc) == TS_ERROR) { + TSError("Error while retrieving client request header\n"); + return false; + } + int method_len = 0; + const char *method = TSHttpHdrMethodGet(req_bufp, req_loc, &method_len); + if (method_len != (int)strlen(TS_HTTP_METHOD_POST) || strncasecmp(method, TS_HTTP_METHOD_POST, method_len) != 0) { + TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); + return false; + } + TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); + return true; +} + +static int +global_plugin(TSCont contp ATS_UNUSED, TSEvent event, void *edata) +{ + TSDebug(PLUGIN_NAME, "transform_plugin starting"); + TSHttpTxn txnp = (TSHttpTxn)edata; + + switch (event) { + case TS_EVENT_HTTP_READ_REQUEST_HDR: + if (is_post_request(txnp)) { + TSHttpTxnConfigIntSet(txnp, TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, 1); + TSHttpTxnHookAdd(txnp, TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, TSContCreate(request_buffer_plugin, TSMutexCreate())); + } + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; + default: + break; + } + + return 0; +} + +void +TSPluginInit(int argc ATS_UNUSED, const char *argv[] ATS_UNUSED) +{ + TSPluginRegistrationInfo info; + + info.plugin_name = PLUGIN_NAME; + info.vendor_name = "Apache Software Foundation"; + info.support_email = "dev@trafficserver.apache.org"; + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSDebug(PLUGIN_NAME, "[%s] Plugin registration failed", PLUGIN_NAME); + + goto Lerror; + } + + /* This is call we could use if we need to protect global data */ + /* TSReleaseAssert ((mutex = TSMutexCreate()) != TS_NULL_MUTEX); */ + + TSMutex mutex = TS_NULL_MUTEX; + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(global_plugin, mutex)); + TSDebug(PLUGIN_NAME, "[%s] Plugin registration succeeded", PLUGIN_NAME); + return; + +Lerror: + TSDebug(PLUGIN_NAME, "[%s] Plugin disabled", PLUGIN_NAME); +} diff --git a/lib/ts/apidefs.h.in b/lib/ts/apidefs.h.in index a34d2dc3231..b0621bd1eff 100644 --- a/lib/ts/apidefs.h.in +++ b/lib/ts/apidefs.h.in @@ -293,6 +293,7 @@ typedef enum { TS_SSL_SERVER_VERIFY_HOOK, TS_SSL_SESSION_HOOK, TS_SSL_LAST_HOOK = TS_SSL_SESSION_HOOK, + TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK = 23, TS_HTTP_LAST_HOOK } TSHttpHookID; @@ -451,6 +452,7 @@ typedef enum { TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60022, TS_EVENT_VCONN_PRE_ACCEPT = 60023, TS_EVENT_LIFECYCLE_MSG = 60024, + TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE = 60025, TS_EVENT_MGMT_UPDATE = 60100, TS_EVENT_INTERNAL_60200 = 60200, TS_EVENT_INTERNAL_60201 = 60201, @@ -764,6 +766,7 @@ typedef enum { TS_CONFIG_HTTP_NORMALIZE_AE, TS_CONFIG_HTTP_INSERT_FORWARDED, TS_CONFIG_HTTP_ALLOW_MULTI_RANGE, + TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, TS_CONFIG_LAST_ENTRY } TSOverridableConfigKey; diff --git a/plugins/experimental/ts_lua/ts_lua_http_config.c b/plugins/experimental/ts_lua/ts_lua_http_config.c index 8bf78c7ec73..c4b75931985 100644 --- a/plugins/experimental/ts_lua/ts_lua_http_config.c +++ b/plugins/experimental/ts_lua/ts_lua_http_config.c @@ -133,6 +133,7 @@ typedef enum { TS_LUA_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS = TS_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS, TS_LUA_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT = TS_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT, TS_LUA_CONFIG_HTTP_ALLOW_MULTI_RANGE = TS_CONFIG_HTTP_ALLOW_MULTI_RANGE, + TS_LUA_CONFIG_HTTP_REQUEST_BUFFER_ENABLED = TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, TS_LUA_CONFIG_LAST_ENTRY = TS_CONFIG_LAST_ENTRY, } TSLuaOverridableConfigKey; @@ -258,6 +259,7 @@ ts_lua_var_item ts_lua_http_config_vars[] = { TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PER_PARENT_CONNECT_ATTEMPTS), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PARENT_CONNECT_ATTEMPT_TIMEOUT), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_ALLOW_MULTI_RANGE), + TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_REQUEST_BUFFER_ENABLED), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_LAST_ENTRY), }; diff --git a/proxy/InkAPI.cc b/proxy/InkAPI.cc index 176c6e6a5f7..d82abdba12f 100644 --- a/proxy/InkAPI.cc +++ b/proxy/InkAPI.cc @@ -8114,6 +8114,9 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr case TS_CONFIG_HTTP_POST_CHECK_CONTENT_LENGTH_ENABLED: ret = _memberp_to_generic(&overridableHttpConfig->post_check_content_length_enabled, typep); break; + case TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED: + ret = _memberp_to_generic(&overridableHttpConfig->request_buffer_enabled, typep); + break; case TS_CONFIG_HTTP_GLOBAL_USER_AGENT_HEADER: ret = _memberp_to_generic(&overridableHttpConfig->global_user_agent_header, typep); break; @@ -8594,6 +8597,8 @@ TSHttpTxnConfigFind(const char *name, int length, TSOverridableConfigKey *conf, case 'd': if (!strncmp(name, "proxy.config.http.forward_connect_method", length)) { cnf = TS_CONFIG_HTTP_FORWARD_CONNECT_METHOD; + } else if (!strncmp(name, "proxy.config.http.request_buffer_enabled", length)) { + cnf = TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED; } break; case 'e': @@ -9629,3 +9634,11 @@ TSRemapToUrlGet(TSHttpTxn txnp, TSMLoc *urlLocp) { return remapUrlGet(txnp, urlLocp, &UrlMappingContainer::getToURL); } + +tsapi TSIOBufferReader +TSHttpTxnPostBufferReaderGet(TSHttpTxn txnp) +{ + sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); + HttpSM *sm = (HttpSM *)txnp; + return (TSIOBufferReader)sm->get_postbuf_clone_reader(); +} diff --git a/proxy/InkAPITest.cc b/proxy/InkAPITest.cc index 3f62d1fdb36..9aba8d0be1f 100644 --- a/proxy/InkAPITest.cc +++ b/proxy/InkAPITest.cc @@ -5545,7 +5545,8 @@ typedef enum { ORIG_TS_SSL_SERVERNAME_HOOK, ORIG_TS_SSL_SERVER_VERIFY_HOOK, ORIG_TS_SSL_SESSION_HOOK, - ORIG_TS_SSL_LAST_HOOK = ORIG_TS_SSL_SESSION_HOOK, + ORIG_TS_SSL_LAST_HOOK = ORIG_TS_SSL_SESSION_HOOK, + ORIG_TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK = 23, ORIG_TS_HTTP_LAST_HOOK } ORIG_TSHttpHookID; @@ -7603,7 +7604,8 @@ const char *SDK_Overridable_Configs[TS_CONFIG_LAST_ENTRY] = {"proxy.config.url_r "proxy.config.http.parent_proxy.connect_attempts_timeout", "proxy.config.http.normalize_ae", "proxy.config.http.insert_forwarded", - "proxy.config.http.allow_multi_range"}; + "proxy.config.http.allow_multi_range", + "proxy.config.http.request_buffer_enabled"}; REGRESSION_TEST(SDK_API_OVERRIDABLE_CONFIGS)(RegressionTest *test, int /* atype ATS_UNUSED */, int *pstatus) { diff --git a/proxy/api/ts/ts.h b/proxy/api/ts/ts.h index 7b1b75203ce..2f07c641ae1 100644 --- a/proxy/api/ts/ts.h +++ b/proxy/api/ts/ts.h @@ -2460,6 +2460,11 @@ tsapi TSReturnCode TSRemapFromUrlGet(TSHttpTxn txnp, TSMLoc *urlLocp); // tsapi TSReturnCode TSRemapToUrlGet(TSHttpTxn txnp, TSMLoc *urlLocp); +/* + * Get a TSIOBufferReader to read the buffered body. The return value needs to be freed. + */ +tsapi TSIOBufferReader TSHttpTxnPostBufferReaderGet(TSHttpTxn txnp); + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index a5fce9cee9a..f802fc088ae 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -966,6 +966,7 @@ HttpConfig::startup() HttpEstablishStaticConfigLongLong(c.oride.flow_high_water_mark, "proxy.config.http.flow_control.high_water"); HttpEstablishStaticConfigLongLong(c.oride.flow_low_water_mark, "proxy.config.http.flow_control.low_water"); HttpEstablishStaticConfigByte(c.oride.post_check_content_length_enabled, "proxy.config.http.post.check.content_length.enabled"); + HttpEstablishStaticConfigByte(c.oride.request_buffer_enabled, "proxy.config.http.request_buffer_enabled"); HttpEstablishStaticConfigByte(c.strict_uri_parsing, "proxy.config.http.strict_uri_parsing"); // [amc] This is a bit of a mess, need to figure out to make this cleaner. @@ -1247,6 +1248,8 @@ HttpConfig::reconfigure() params->oride.post_check_content_length_enabled = INT_TO_BOOL(m_master.oride.post_check_content_length_enabled); + params->oride.request_buffer_enabled = INT_TO_BOOL(m_master.oride.request_buffer_enabled); + params->oride.flow_control_enabled = INT_TO_BOOL(m_master.oride.flow_control_enabled); params->oride.flow_high_water_mark = m_master.oride.flow_high_water_mark; params->oride.flow_low_water_mark = m_master.oride.flow_low_water_mark; diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index a01a2d95e16..8a22a4396f9 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -450,6 +450,7 @@ struct OverridableHttpConfigParams { parent_failures_update_hostdb(0), cache_open_write_fail_action(0), post_check_content_length_enabled(1), + request_buffer_enabled(0), ssl_client_verify_server(0), redirect_use_orig_cache_key(0), number_of_redirections(0), @@ -624,6 +625,11 @@ struct OverridableHttpConfigParams { //////////////////////// MgmtByte post_check_content_length_enabled; + //////////////////////////////////////////////// + // Buffer post body before connecting servers // + //////////////////////////////////////////////// + MgmtByte request_buffer_enabled; + ///////////////////////////// // server verification mode// ///////////////////////////// diff --git a/proxy/http/HttpDebugNames.cc b/proxy/http/HttpDebugNames.cc index b72a50c8adc..b17083e3a2a 100644 --- a/proxy/http/HttpDebugNames.cc +++ b/proxy/http/HttpDebugNames.cc @@ -352,6 +352,10 @@ HttpDebugNames::get_action_name(HttpTransact::StateMachineAction_t e) return ("SM_ACTION_DRAIN_REQUEST_BODY"); #endif /* PROXY_DRAIN */ + case HttpTransact::SM_ACTION_WAIT_FOR_FULL_BODY: + return ("SM_ACTION_WAIT_FOR_FULL_BODY"); + case HttpTransact::SM_ACTION_REQUEST_BUFFER_READ_COMPLETE: + return ("SM_ACTION_REQUEST_BUFFER_READ_COMPLETE"); case HttpTransact::SM_ACTION_API_SM_START: return ("SM_ACTION_API_SM_START"); case HttpTransact::SM_ACTION_REDIRECT_READ: @@ -438,6 +442,8 @@ HttpDebugNames::get_api_hook_name(TSHttpHookID t) return "TS_HTTP_SEND_RESPONSE_HDR_HOOK"; case TS_HTTP_REQUEST_TRANSFORM_HOOK: return "TS_HTTP_REQUEST_TRANSFORM_HOOK"; + case TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK: + return "TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK"; case TS_HTTP_RESPONSE_TRANSFORM_HOOK: return "TS_HTTP_RESPONSE_TRANSFORM_HOOK"; case TS_HTTP_SELECT_ALT_HOOK: diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 3a96156b6cf..b620f85c895 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -838,6 +838,48 @@ HttpSM::state_drain_client_request_body(int event, void *data) } #endif /* PROXY_DRAIN */ +void +HttpSM::wait_for_full_body() +{ + is_waiting_for_full_body = true; + HTTP_SM_SET_DEFAULT_HANDLER(&HttpSM::tunnel_handler_post); + bool chunked = (t_state.client_info.transfer_encoding == HttpTransact::CHUNKED_ENCODING); + int64_t alloc_index; + HttpTunnelProducer *p = nullptr; + + // content length is undefined, use default buffer size + if (t_state.hdr_info.request_content_length == HTTP_UNDEFINED_CL) { + alloc_index = (int)t_state.txn_conf->default_buffer_size_index; + if (alloc_index < MIN_CONFIG_BUFFER_SIZE_INDEX || alloc_index > MAX_BUFFER_SIZE_INDEX) { + alloc_index = DEFAULT_REQUEST_BUFFER_SIZE_INDEX; + } + } else { + alloc_index = buffer_size_to_index(t_state.hdr_info.request_content_length); + } + MIOBuffer *post_buffer = new_MIOBuffer(alloc_index); + IOBufferReader *buf_start = post_buffer->alloc_reader(); + + this->_postbuf.init(post_buffer->clone_reader(buf_start)); + + // Note: Many browsers, Netscape and IE included send two extra + // bytes (CRLF) at the end of the post. We just ignore those + // bytes since the sending them is not spec + + // Next order of business if copy the remaining data from the + // header buffer into new buffer + int64_t post_bytes = chunked ? INT64_MAX : t_state.hdr_info.request_content_length; + client_request_body_bytes = post_buffer->write(ua_buffer_reader, chunked ? ua_buffer_reader->read_avail() : post_bytes); + + ua_buffer_reader->consume(client_request_body_bytes); + p = tunnel.add_producer(ua_entry->vc, post_bytes, buf_start, &HttpSM::tunnel_handler_post_ua, HT_BUFFER_READ, "ua post buffer"); + if (chunked) { + tunnel.set_producer_chunking_action(p, 0, TCA_PASSTHRU_CHUNKED_CONTENT); + } + ua_entry->in_tunnel = true; + ua_txn->set_inactivity_timeout(HRTIME_SECONDS(t_state.txn_conf->transaction_no_activity_timeout_in)); + tunnel.tunnel_run(p); +} + int HttpSM::state_watch_for_client_abort(int event, void *data) { @@ -1601,6 +1643,7 @@ HttpSM::handle_api_return() case HttpTransact::SM_ACTION_API_PRE_REMAP: case HttpTransact::SM_ACTION_API_POST_REMAP: case HttpTransact::SM_ACTION_API_READ_REQUEST_HDR: + case HttpTransact::SM_ACTION_REQUEST_BUFFER_READ_COMPLETE: case HttpTransact::SM_ACTION_API_OS_DNS: case HttpTransact::SM_ACTION_API_READ_RESPONSE_HDR: call_transact_and_set_next_state(nullptr); @@ -2616,7 +2659,7 @@ HttpSM::main_handler(int event, void *data) void HttpSM::tunnel_handler_post_or_put(HttpTunnelProducer *p) { - ink_assert(p->vc_type == HT_HTTP_CLIENT); + ink_assert(p->vc_type == HT_HTTP_CLIENT || (p->handler_state == HTTP_SM_POST_UA_FAIL && p->vc_type == HT_BUFFER_READ)); HttpTunnelConsumer *c; // If there is a post transform, remove it's entry from the State @@ -2715,7 +2758,12 @@ HttpSM::tunnel_handler_post(int event, void *data) // The tunnel calls this when it is done int p_handler_state = p->handler_state; - tunnel_handler_post_or_put(p); + if (is_waiting_for_full_body && !this->is_postbuf_valid()) { + p_handler_state = HTTP_SM_POST_SERVER_FAIL; + } + if (p->vc_type != HT_BUFFER_READ) { + tunnel_handler_post_or_put(p); + } switch (p_handler_state) { case HTTP_SM_POST_SERVER_FAIL: @@ -2725,6 +2773,14 @@ HttpSM::tunnel_handler_post(int event, void *data) break; case HTTP_SM_POST_SUCCESS: // It's time to start reading the response + if (is_waiting_for_full_body) { + is_waiting_for_full_body = false; + is_using_post_buffer = true; + client_request_body_bytes = this->postbuf_buffer_avail(); + + call_transact_and_set_next_state(HttpTransact::HandleRequestBufferDone); + break; + } setup_server_read_response_header(); break; default: @@ -3462,7 +3518,7 @@ HttpSM::tunnel_handler_post_ua(int event, HttpTunnelProducer *p) // we were setting it again to true but incorrectly in // the case of a transform hsm_release_assert(ua_entry->in_tunnel == true); - if (p->consumer_list.head->vc_type == HT_TRANSFORM) { + if (p->consumer_list.head && p->consumer_list.head->vc_type == HT_TRANSFORM) { hsm_release_assert(post_transform_info.entry->in_tunnel == true); } else if (server_entry != nullptr) { hsm_release_assert(server_entry->in_tunnel == true); @@ -3482,6 +3538,7 @@ HttpSM::tunnel_handler_post_ua(int event, HttpTunnelProducer *p) tunnel.local_finish_all(p); } } + // Initiate another read to watch catch aborts and // timeouts ua_entry->vc_handler = &HttpSM::state_watch_for_client_abort; @@ -3507,6 +3564,7 @@ HttpSM::tunnel_handler_for_partial_post(int event, void * /* data ATS_UNUSED */) tunnel.reset(); t_state.redirect_info.redirect_in_process = false; + is_using_post_buffer = false; if (post_failed) { post_failed = false; @@ -5062,6 +5120,9 @@ HttpSM::do_api_callout_internal() case HttpTransact::SM_ACTION_API_READ_REQUEST_HDR: cur_hook_id = TS_HTTP_READ_REQUEST_HDR_HOOK; break; + case HttpTransact::SM_ACTION_REQUEST_BUFFER_READ_COMPLETE: + cur_hook_id = TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK; + break; case HttpTransact::SM_ACTION_API_OS_DNS: cur_hook_id = TS_HTTP_OS_DNS_HOOK; break; @@ -5302,8 +5363,12 @@ HttpSM::handle_post_failure() STATE_ENTER(&HttpSM::handle_post_failure, VC_EVENT_NONE); ink_assert(ua_entry->vc == ua_txn); - ink_assert(server_entry->eos == true); + ink_assert(is_waiting_for_full_body || server_entry->eos == true); + if (is_waiting_for_full_body) { + call_transact_and_set_next_state(HttpTransact::Forbidden); + return; + } // First order of business is to clean up from // the tunnel // note: since the tunnel is providing the buffer for a lingering @@ -5582,7 +5647,8 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) // YTS Team, yamsat Plugin // if redirect_in_process and redirection is enabled add static producer - if (t_state.redirect_info.redirect_in_process && enable_redirection && (this->_postbuf.postdata_copy_buffer_start != nullptr)) { + if (is_using_post_buffer || + (t_state.redirect_info.redirect_in_process && enable_redirection && this->_postbuf.postdata_copy_buffer_start != nullptr)) { post_redirect = true; // copy the post data into a new producer buffer for static producer MIOBuffer *postdata_producer_buffer = new_empty_MIOBuffer(); @@ -7172,6 +7238,7 @@ HttpSM::set_next_state() case HttpTransact::SM_ACTION_API_PRE_REMAP: case HttpTransact::SM_ACTION_API_POST_REMAP: case HttpTransact::SM_ACTION_API_READ_REQUEST_HDR: + case HttpTransact::SM_ACTION_REQUEST_BUFFER_READ_COMPLETE: case HttpTransact::SM_ACTION_API_OS_DNS: case HttpTransact::SM_ACTION_API_SEND_REQUEST_HDR: case HttpTransact::SM_ACTION_API_READ_CACHE_HDR: @@ -7582,6 +7649,11 @@ HttpSM::set_next_state() } #endif /* PROXY_DRAIN */ + case HttpTransact::SM_ACTION_WAIT_FOR_FULL_BODY: { + wait_for_full_body(); + break; + } + case HttpTransact::SM_ACTION_CONTINUE: { ink_release_assert(!"Not implemented"); break; @@ -8027,12 +8099,21 @@ HttpSM::find_proto_string(HTTPVersion version) const void PostDataBuffers::copy_partial_post_data() { - this->postdata_copy_buffer->write(this->ua_buffer_reader); + if (post_data_buffer_done) { + return; + } Debug("http_redirect", "[PostDataBuffers::copy_partial_post_data] wrote %" PRId64 " bytes to buffers %" PRId64 "", this->ua_buffer_reader->read_avail(), this->postdata_copy_buffer_start->read_avail()); + this->postdata_copy_buffer->write(this->ua_buffer_reader); this->ua_buffer_reader->consume(this->ua_buffer_reader->read_avail()); } +IOBufferReader * +PostDataBuffers::get_post_data_buffer_clone_reader() +{ + return this->postdata_copy_buffer->clone_reader(this->postdata_copy_buffer_start); +} + // YTS Team, yamsat Plugin // Allocating the post data buffers void @@ -8043,6 +8124,7 @@ PostDataBuffers::init(IOBufferReader *ua_reader) this->ua_buffer_reader = ua_reader; if (this->postdata_copy_buffer == nullptr) { + this->post_data_buffer_done = false; ink_assert(this->postdata_copy_buffer_start == nullptr); this->postdata_copy_buffer = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_4K); this->postdata_copy_buffer_start = this->postdata_copy_buffer->alloc_reader(); @@ -8063,6 +8145,7 @@ PostDataBuffers::clear() this->postdata_copy_buffer = nullptr; this->postdata_copy_buffer_start = nullptr; // deallocated by the buffer } + this->post_data_buffer_done = false; } PostDataBuffers::~PostDataBuffers() diff --git a/proxy/http/HttpSM.h b/proxy/http/HttpSM.h index d98d3a4f5be..4cb535b9e5e 100644 --- a/proxy/http/HttpSM.h +++ b/proxy/http/HttpSM.h @@ -184,10 +184,27 @@ class PostDataBuffers MIOBuffer *postdata_copy_buffer = nullptr; IOBufferReader *postdata_copy_buffer_start = nullptr; IOBufferReader *ua_buffer_reader = nullptr; + bool post_data_buffer_done = false; void clear(); void init(IOBufferReader *ua_reader); void copy_partial_post_data(); + IOBufferReader *get_post_data_buffer_clone_reader(); + void + set_post_data_buffer_done(bool done) + { + post_data_buffer_done = done; + } + bool + get_post_data_buffer_done() + { + return post_data_buffer_done; + } + bool + is_valid() + { + return postdata_copy_buffer_start != nullptr; + } ~PostDataBuffers(); }; @@ -318,6 +335,10 @@ class HttpSM : public Continuation void disable_redirect(); void postbuf_copy_partial_data(); void postbuf_init(IOBufferReader *ua_reader); + void set_postbuf_done(bool done); + bool get_postbuf_done(); + bool is_postbuf_valid(); + IOBufferReader *get_postbuf_clone_reader(); protected: int reentrancy_count = 0; @@ -454,6 +475,8 @@ class HttpSM : public Continuation void do_drain_request_body(); #endif + void wait_for_full_body(); + virtual void handle_api_return(); void handle_server_setup_error(int event, void *data); void handle_http_server_open(); @@ -528,6 +551,8 @@ class HttpSM : public Continuation const char *client_cipher_suite = "-"; int server_transact_count = 0; bool server_connection_is_ssl = false; + bool is_waiting_for_full_body = false; + bool is_using_post_buffer = false; TransactionMilestones milestones; ink_hrtime api_timer = 0; @@ -731,4 +756,27 @@ HttpSM::postbuf_init(IOBufferReader *ua_reader) this->_postbuf.init(ua_reader); } +inline void +HttpSM::set_postbuf_done(bool done) +{ + this->_postbuf.set_post_data_buffer_done(done); +} + +inline bool +HttpSM::get_postbuf_done() +{ + return this->_postbuf.get_post_data_buffer_done(); +} + +inline bool +HttpSM::is_postbuf_valid() +{ + return this->_postbuf.is_valid(); +} + +inline IOBufferReader * +HttpSM::get_postbuf_clone_reader() +{ + return this->_postbuf.get_post_data_buffer_clone_reader(); +} #endif diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc index 0c5ae6b9222..9deb7338243 100644 --- a/proxy/http/HttpTransact.cc +++ b/proxy/http/HttpTransact.cc @@ -1121,77 +1121,81 @@ HttpTransact::HandleRequest(State *s) { TxnDebug("http_trans", "START HttpTransact::HandleRequest"); - ink_assert(!s->hdr_info.server_request.valid()); + if (!s->request_data.hdr) { + ink_assert(!s->hdr_info.server_request.valid()); - HTTP_INCREMENT_DYN_STAT(http_incoming_requests_stat); + HTTP_INCREMENT_DYN_STAT(http_incoming_requests_stat); - if (s->client_info.port_attribute == HttpProxyPort::TRANSPORT_SSL) { - HTTP_INCREMENT_DYN_STAT(https_incoming_requests_stat); - } - - /////////////////////////////////////////////// - // if request is bad, return error response // - /////////////////////////////////////////////// - - if (!(is_request_valid(s, &s->hdr_info.client_request))) { - HTTP_INCREMENT_DYN_STAT(http_invalid_client_requests_stat); - TxnDebug("http_seq", "[HttpTransact::HandleRequest] request invalid."); - s->next_action = SM_ACTION_SEND_ERROR_CACHE_NOOP; - // s->next_action = HttpTransact::PROXY_INTERNAL_CACHE_NOOP; - return; - } - TxnDebug("http_seq", "[HttpTransact::HandleRequest] request valid."); + if (s->client_info.port_attribute == HttpProxyPort::TRANSPORT_SSL) { + HTTP_INCREMENT_DYN_STAT(https_incoming_requests_stat); + } - if (is_debug_tag_set("http_chdr_describe")) { - obj_describe(s->hdr_info.client_request.m_http, true); - } + /////////////////////////////////////////////// + // if request is bad, return error response // + /////////////////////////////////////////////// - // at this point we are guaranteed that the request is good and acceptable. - // initialize some state variables from the request (client version, - // client keep-alive, cache action, etc. - initialize_state_variables_from_request(s, &s->hdr_info.client_request); + if (!(is_request_valid(s, &s->hdr_info.client_request))) { + HTTP_INCREMENT_DYN_STAT(http_invalid_client_requests_stat); + TxnDebug("http_seq", "[HttpTransact::HandleRequest] request invalid."); + s->next_action = SM_ACTION_SEND_ERROR_CACHE_NOOP; + // s->next_action = HttpTransact::PROXY_INTERNAL_CACHE_NOOP; + return; + } + TxnDebug("http_seq", "[HttpTransact::HandleRequest] request valid."); + + if (is_debug_tag_set("http_chdr_describe")) { + obj_describe(s->hdr_info.client_request.m_http, true); + } + // at this point we are guaranteed that the request is good and acceptable. + // initialize some state variables from the request (client version, + // client keep-alive, cache action, etc. + initialize_state_variables_from_request(s, &s->hdr_info.client_request); + // The following chunk of code will limit the maximum number of websocket connections (TS-3659) + if (s->is_upgrade_request && s->is_websocket && s->http_config_param->max_websocket_connections >= 0) { + int64_t val = 0; + HTTP_READ_DYN_SUM(http_websocket_current_active_client_connections_stat, val); + if (val >= s->http_config_param->max_websocket_connections) { + s->is_websocket = false; // unset to avoid screwing up stats. + TxnDebug("http_trans", "Rejecting websocket connection because the limit has been exceeded"); + bootstrap_state_variables_from_request(s, &s->hdr_info.client_request); + build_error_response(s, HTTP_STATUS_SERVICE_UNAVAILABLE, "WebSocket Connection Limit Exceeded", nullptr); + TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); + } + } - // The following chunk of code will limit the maximum number of websocket connections (TS-3659) - if (s->is_upgrade_request && s->is_websocket && s->http_config_param->max_websocket_connections >= 0) { - int64_t val = 0; - HTTP_READ_DYN_SUM(http_websocket_current_active_client_connections_stat, val); - if (val >= s->http_config_param->max_websocket_connections) { - s->is_websocket = false; // unset to avoid screwing up stats. - TxnDebug("http_trans", "Rejecting websocket connection because the limit has been exceeded"); + // The following code is configurable to allow a user to control the max post size (TS-3631) + if (s->http_config_param->max_post_size > 0 && s->hdr_info.request_content_length > 0 && + s->hdr_info.request_content_length > s->http_config_param->max_post_size) { + TxnDebug("http_trans", "Max post size %" PRId64 " Client tried to post a body that was too large.", + s->http_config_param->max_post_size); + HTTP_INCREMENT_DYN_STAT(http_post_body_too_large); bootstrap_state_variables_from_request(s, &s->hdr_info.client_request); - build_error_response(s, HTTP_STATUS_SERVICE_UNAVAILABLE, "WebSocket Connection Limit Exceeded", nullptr); + build_error_response(s, HTTP_STATUS_REQUEST_ENTITY_TOO_LARGE, "Request Entity Too Large", "request#entity_too_large"); + s->squid_codes.log_code = SQUID_LOG_ERR_POST_ENTITY_TOO_LARGE; TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); } - } - - // The following code is configurable to allow a user to control the max post size (TS-3631) - if (s->http_config_param->max_post_size > 0 && s->hdr_info.request_content_length > 0 && - s->hdr_info.request_content_length > s->http_config_param->max_post_size) { - TxnDebug("http_trans", "Max post size %" PRId64 " Client tried to post a body that was too large.", - s->http_config_param->max_post_size); - HTTP_INCREMENT_DYN_STAT(http_post_body_too_large); - bootstrap_state_variables_from_request(s, &s->hdr_info.client_request); - build_error_response(s, HTTP_STATUS_REQUEST_ENTITY_TOO_LARGE, "Request Entity Too Large", "request#entity_too_large"); - s->squid_codes.log_code = SQUID_LOG_ERR_POST_ENTITY_TOO_LARGE; - TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); - } - // The following chunk of code allows you to disallow post w/ expect 100-continue (TS-3459) - if (s->hdr_info.request_content_length && s->http_config_param->disallow_post_100_continue) { - MIMEField *expect = s->hdr_info.client_request.field_find(MIME_FIELD_EXPECT, MIME_LEN_EXPECT); - - if (expect != nullptr) { - const char *expect_hdr_val = nullptr; - int expect_hdr_val_len = 0; - expect_hdr_val = expect->value_get(&expect_hdr_val_len); - if (ptr_len_casecmp(expect_hdr_val, expect_hdr_val_len, HTTP_VALUE_100_CONTINUE, HTTP_LEN_100_CONTINUE) == 0) { - // Let's error out this request. - TxnDebug("http_trans", "Client sent a post expect: 100-continue, sending 405."); - HTTP_INCREMENT_DYN_STAT(disallowed_post_100_continue); - build_error_response(s, HTTP_STATUS_METHOD_NOT_ALLOWED, "Method Not Allowed", "request#method_unsupported"); - TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); + // The following chunk of code allows you to disallow post w/ expect 100-continue (TS-3459) + if (s->hdr_info.request_content_length && s->http_config_param->disallow_post_100_continue) { + MIMEField *expect = s->hdr_info.client_request.field_find(MIME_FIELD_EXPECT, MIME_LEN_EXPECT); + + if (expect != nullptr) { + const char *expect_hdr_val = nullptr; + int expect_hdr_val_len = 0; + expect_hdr_val = expect->value_get(&expect_hdr_val_len); + if (ptr_len_casecmp(expect_hdr_val, expect_hdr_val_len, HTTP_VALUE_100_CONTINUE, HTTP_LEN_100_CONTINUE) == 0) { + // Let's error out this request. + TxnDebug("http_trans", "Client sent a post expect: 100-continue, sending 405."); + HTTP_INCREMENT_DYN_STAT(disallowed_post_100_continue); + build_error_response(s, HTTP_STATUS_METHOD_NOT_ALLOWED, "Method Not Allowed", "request#method_unsupported"); + TRANSACT_RETURN(SM_ACTION_SEND_ERROR_CACHE_NOOP, nullptr); + } } } + if (s->txn_conf->request_buffer_enabled && + (s->hdr_info.request_content_length > 0 || s->client_info.transfer_encoding == CHUNKED_ENCODING)) { + TRANSACT_RETURN(SM_ACTION_WAIT_FOR_FULL_BODY, nullptr); + } } // Cache lookup or not will be decided later at DecideCacheLookup(). @@ -1304,6 +1308,12 @@ HttpTransact::HandleRequest(State *s) } } +void +HttpTransact::HandleRequestBufferDone(State *s) +{ + TRANSACT_RETURN(SM_ACTION_REQUEST_BUFFER_READ_COMPLETE, HttpTransact::HandleRequest); +} + void HttpTransact::setup_plugin_request_intercept(State *s) { diff --git a/proxy/http/HttpTransact.h b/proxy/http/HttpTransact.h index 3b09a713754..3d300585870 100644 --- a/proxy/http/HttpTransact.h +++ b/proxy/http/HttpTransact.h @@ -425,6 +425,8 @@ class HttpTransact SM_ACTION_DRAIN_REQUEST_BODY, #endif /* PROXY_DRAIN */ + SM_ACTION_WAIT_FOR_FULL_BODY, + SM_ACTION_REQUEST_BUFFER_READ_COMPLETE, SM_ACTION_SERVE_FROM_CACHE, SM_ACTION_SERVER_READ, SM_ACTION_SERVER_PARSE_NEXT_HDR, @@ -957,6 +959,7 @@ class HttpTransact static void PerformRemap(State *s); static void ModifyRequest(State *s); static void HandleRequest(State *s); + static void HandleRequestBufferDone(State *s); static bool handleIfRedirect(State *s); static void StartAccessControl(State *s); diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index c5f1fa5e832..606e1b00a0e 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -757,7 +757,6 @@ void HttpTunnel::tunnel_run(HttpTunnelProducer *p_arg) { Debug("http_tunnel", "tunnel_run started, p_arg is %s", p_arg ? "provided" : "NULL"); - if (p_arg) { producer_run(p_arg); } else { @@ -876,7 +875,6 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) // Do the IO on the consumers first so // data doesn't disappear out from // under the tunnel - ink_release_assert(p->num_consumers > 0); for (c = p->consumer_list.head; c;) { // Create a reader for each consumer. The reader allows // us to implement skip bytes @@ -952,15 +950,20 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) // YTS Team, yamsat Plugin // Allocate and copy partial POST data to buffers. Check for the various parameters // including the maximum configured post data size - if (p->alive && sm->t_state.method == HTTP_WKSIDX_POST && sm->enable_redirection && (p->vc_type == HT_HTTP_CLIENT)) { + if ((p->vc_type == HT_BUFFER_READ && sm->is_postbuf_valid()) || + (p->alive && sm->t_state.method == HTTP_WKSIDX_POST && sm->enable_redirection && p->vc_type == HT_HTTP_CLIENT)) { Debug("http_redirect", "[HttpTunnel::producer_run] client post: %" PRId64 " max size: %" PRId64 "", p->buffer_start->read_avail(), HttpConfig::m_master.post_copy_size); // (note that since we are not dechunking POST, this is the chunked size if chunked) if (p->buffer_start->read_avail() > HttpConfig::m_master.post_copy_size) { - Debug("http_redirect", "[HttpTunnel::producer_handler] post exceeds buffer limit, buffer_avail=%" PRId64 " limit=%" PRId64 "", - p->buffer_start->read_avail(), HttpConfig::m_master.post_copy_size); + Warning("http_redirect, [HttpTunnel::producer_handler] post exceeds buffer limit, buffer_avail=%" PRId64 " limit=%" PRId64 "", + p->buffer_start->read_avail(), HttpConfig::m_master.post_copy_size); sm->disable_redirect(); + if (p->vc_type == HT_BUFFER_READ) { + producer_handler(VC_EVENT_ERROR, p); + return; + } } else { sm->postbuf_copy_partial_data(); } @@ -993,8 +996,7 @@ HttpTunnel::producer_run(HttpTunnelProducer *p) // p->chunked_handler.skip_bytes); producer_handler(VC_EVENT_READ_READY, p); - if (!p->chunked_handler.chunked_reader->read_avail() && sm->redirection_tries > 0 && - p->vc_type == HT_HTTP_CLIENT) { // read_avail() == 0 + if (sm->get_postbuf_done() && p->vc_type == HT_HTTP_CLIENT) { // read_avail() == 0 // [bug 2579251] // Ugh, this is horrible but in the redirect case they are running a the tunnel again with the // now closed/empty producer to trigger PRECOMPLETE. If the POST was chunked, producer_n is set @@ -1162,17 +1164,24 @@ HttpTunnel::producer_handler(int event, HttpTunnelProducer *p) // YTS Team, yamsat Plugin // Copy partial POST data to buffers. Check for the various parameters including // the maximum configured post data size - if (sm->t_state.method == HTTP_WKSIDX_POST && sm->enable_redirection && - (event == VC_EVENT_READ_READY || event == VC_EVENT_READ_COMPLETE) && (p->vc_type == HT_HTTP_CLIENT)) { + if ((p->vc_type == HT_BUFFER_READ && sm->is_postbuf_valid()) || + (sm->t_state.method == HTTP_WKSIDX_POST && sm->enable_redirection && + (event == VC_EVENT_READ_READY || event == VC_EVENT_READ_COMPLETE) && p->vc_type == HT_HTTP_CLIENT)) { Debug("http_redirect", "[HttpTunnel::producer_handler] [%s %s]", p->name, HttpDebugNames::get_event_name(event)); if ((sm->postbuf_buffer_avail() + sm->postbuf_reader_avail()) > HttpConfig::m_master.post_copy_size) { - Debug("http_redirect", "[HttpTunnel::producer_handler] post exceeds buffer limit, buffer_avail=%" PRId64 - " reader_avail=%" PRId64 " limit=%" PRId64 "", - sm->postbuf_buffer_avail(), sm->postbuf_reader_avail(), HttpConfig::m_master.post_copy_size); + Warning("http_redirect, [HttpTunnel::producer_handler] post exceeds buffer limit, buffer_avail=%" PRId64 + " reader_avail=%" PRId64 " limit=%" PRId64 "", + sm->postbuf_buffer_avail(), sm->postbuf_reader_avail(), HttpConfig::m_master.post_copy_size); sm->disable_redirect(); + if (p->vc_type == HT_BUFFER_READ) { + event = VC_EVENT_ERROR; + } } else { sm->postbuf_copy_partial_data(); + if (event == VC_EVENT_READ_COMPLETE || event == HTTP_TUNNEL_EVENT_PRECOMPLETE || event == VC_EVENT_EOS) { + sm->set_postbuf_done(true); + } } } // end of added logic for partial copy of POST diff --git a/proxy/http/HttpTunnel.h b/proxy/http/HttpTunnel.h index d98c32dd20a..ca152198dfe 100644 --- a/proxy/http/HttpTunnel.h +++ b/proxy/http/HttpTunnel.h @@ -66,14 +66,7 @@ struct HttpTunnelProducer; typedef int (HttpSM::*HttpProducerHandler)(int event, HttpTunnelProducer *p); typedef int (HttpSM::*HttpConsumerHandler)(int event, HttpTunnelConsumer *c); -enum HttpTunnelType_t { - HT_HTTP_SERVER, - HT_HTTP_CLIENT, - HT_CACHE_READ, - HT_CACHE_WRITE, - HT_TRANSFORM, - HT_STATIC, -}; +enum HttpTunnelType_t { HT_HTTP_SERVER, HT_HTTP_CLIENT, HT_CACHE_READ, HT_CACHE_WRITE, HT_TRANSFORM, HT_STATIC, HT_BUFFER_READ }; enum TunnelChunkingAction_t { TCA_CHUNK_CONTENT, From ce0440973279abfa4aa6b60c2c3da657970806a6 Mon Sep 17 00:00:00 2001 From: Zizhong Zhang Date: Tue, 2 Jan 2018 12:29:56 -0800 Subject: [PATCH 2/2] Adding slow_post_test --- tests/gold_tests/slow_post/gold/200.gold | 1 + tests/gold_tests/slow_post/slow_post.test.py | 73 +++++++++ .../gold_tests/slow_post/slow_post_client.py | 59 +++++++ tests/tools/plugins/request_buffer.c | 144 ++++++++++++++++++ 4 files changed, 277 insertions(+) create mode 100644 tests/gold_tests/slow_post/gold/200.gold create mode 100644 tests/gold_tests/slow_post/slow_post.test.py create mode 100644 tests/gold_tests/slow_post/slow_post_client.py create mode 100644 tests/tools/plugins/request_buffer.c diff --git a/tests/gold_tests/slow_post/gold/200.gold b/tests/gold_tests/slow_post/gold/200.gold new file mode 100644 index 00000000000..08839f6bb29 --- /dev/null +++ b/tests/gold_tests/slow_post/gold/200.gold @@ -0,0 +1 @@ +200 diff --git a/tests/gold_tests/slow_post/slow_post.test.py b/tests/gold_tests/slow_post/slow_post.test.py new file mode 100644 index 00000000000..906a87f14fd --- /dev/null +++ b/tests/gold_tests/slow_post/slow_post.test.py @@ -0,0 +1,73 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +class SlowPostAttack: + def __init__(cls): + Test.Summary = 'Test how ATS handles the slow-post attack' + cls._origin_max_connections = 3 + cls._slow_post_client = 'slow_post_client.py' + cls.setupOriginServer() + cls.setupTS() + cls._ts.Setup.CopyAs(cls._slow_post_client, Test.RunDirectory) + + def setupOriginServer(self): + self._server = Test.MakeOriginServer("server") + request_header = {"headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""} + response_header = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} + self._server.addResponse("sessionlog.json", request_header, response_header) + request_header2 = {"headers": "POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\nHost: www.example.com\r\nConnection: keep-alive\r\n\r\n", + "timestamp": "1469733493.993", "body": "a\r\na\r\na\r\n\r\n"} + response_header2 = {"headers": "HTTP/1.1 200 OK\r\nServer: microserver\r\nConnection: close\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} + self._server.addResponse("sessionlog.json", request_header2, response_header2) + + def setupTS(self): + self._ts = Test.MakeATSProcess("ts", select_ports=False) + self._ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(self._server.Variables.Port) + ) + # This plugin can enable request buffer for POST. + self._ts.Disk.plugin_config.AddLine( + 'request_buffer.so' + ) + Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'request_buffer.c'), self._ts) + self._ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'http', + 'proxy.config.http.origin_max_connections': self._origin_max_connections, + # Disable queueing when connection reaches limit + 'proxy.config.http.origin_max_connections_queue': 0, + }) + + def run(self): + tr = Test.AddTestRun() + tr.Processes.Default.Command = 'python3 {0} -p {1} -c {2}'.format( + self._slow_post_client, self._ts.Variables.port, self._origin_max_connections) + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(Test.Processes.ts) + tr.Processes.Default.Streams.stdout = "gold/200.gold" + + +Test.Summary = 'Test how ATS handles the slow-post attack' +slowPostAttack = SlowPostAttack() +slowPostAttack.run() diff --git a/tests/gold_tests/slow_post/slow_post_client.py b/tests/gold_tests/slow_post/slow_post_client.py new file mode 100644 index 00000000000..a132a6792fd --- /dev/null +++ b/tests/gold_tests/slow_post/slow_post_client.py @@ -0,0 +1,59 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +import threading +import requests +import argparse + + +def gen(slow_time): + for _ in range(slow_time): + yield b'a' + time.sleep(1) + + +def slow_post(port, slow_time): + requests.post('http://127.0.0.1:{0}/'.format(port, ), data=gen(slow_time)) + + +def makerequest(port, connection_limit): + client_timeout = 3 + for i in range(connection_limit): + t = threading.Thread(target=slow_post, args=(port, client_timeout + 10)) + t.daemon = True + t.start() + time.sleep(1) + r = requests.get('http://127.0.0.1:{0}/'.format(port,)) + print(r.status_code) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--port", "-p", + type=int, + help="Port to use") + parser.add_argument("--connectionlimit", "-c", + type=int, + help="connection limit") + args = parser.parse_args() + makerequest(args.port, args.connectionlimit) + + +if __name__ == '__main__': + main() diff --git a/tests/tools/plugins/request_buffer.c b/tests/tools/plugins/request_buffer.c new file mode 100644 index 00000000000..7e6df14413c --- /dev/null +++ b/tests/tools/plugins/request_buffer.c @@ -0,0 +1,144 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include +#include +#include + +#include "ts/ts.h" + +#define PLUGIN_NAME "request_buffer" + +#define TS_NULL_MUTEX NULL + +static char * +request_body_get(TSHttpTxn txnp, int *len) +{ + char *ret = NULL; + TSIOBufferReader post_buffer_reader = TSHttpTxnPostBufferReaderGet(txnp); + int64_t read_avail = TSIOBufferReaderAvail(post_buffer_reader); + if (read_avail == 0) { + TSIOBufferReaderFree(post_buffer_reader); + return NULL; + } + + ret = (char *)TSmalloc(sizeof(char) * read_avail); + + int64_t consumed = 0; + int64_t data_len = 0; + const char *char_data = NULL; + TSIOBufferBlock block = TSIOBufferReaderStart(post_buffer_reader); + while (block != NULL) { + char_data = TSIOBufferBlockReadStart(block, post_buffer_reader, &data_len); + memcpy(ret + consumed, char_data, data_len); + consumed += data_len; + block = TSIOBufferBlockNext(block); + } + TSIOBufferReaderFree(post_buffer_reader); + + *len = (int)consumed; + return ret; +} + +static int +request_buffer_plugin(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(PLUGIN_NAME, "request_buffer_plugin starting, event[%d]", event); + TSHttpTxn txnp = (TSHttpTxn)(edata); + if (event == TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE) { + int len = 0; + char *body = request_body_get(txnp, &len); + TSDebug(PLUGIN_NAME, "request_buffer_plugin gets the request body with length[%d]", len); + TSfree(body); + TSContDestroy(contp); + } + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; +} + +bool +is_post_request(TSHttpTxn txnp) +{ + TSMLoc req_loc; + TSMBuffer req_bufp; + if (TSHttpTxnClientReqGet(txnp, &req_bufp, &req_loc) == TS_ERROR) { + TSError("Error while retrieving client request header\n"); + return false; + } + int method_len = 0; + const char *method = TSHttpHdrMethodGet(req_bufp, req_loc, &method_len); + if (method_len != (int)strlen(TS_HTTP_METHOD_POST) || strncasecmp(method, TS_HTTP_METHOD_POST, method_len) != 0) { + TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); + return false; + } + TSHandleMLocRelease(req_bufp, TS_NULL_MLOC, req_loc); + return true; +} + +static int +global_plugin(TSCont contp, TSEvent event, void *edata) +{ + TSDebug(PLUGIN_NAME, "transform_plugin starting"); + TSHttpTxn txnp = (TSHttpTxn)edata; + + switch (event) { + case TS_EVENT_HTTP_READ_REQUEST_HDR: + if (is_post_request(txnp)) { + TSHttpTxnConfigIntSet(txnp, TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, 1); + TSHttpTxnHookAdd(txnp, TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, TSContCreate(request_buffer_plugin, TSMutexCreate())); + } + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; + default: + break; + } + + return 0; +} + +void +TSPluginInit(int argc, const char *argv[]) +{ + TSPluginRegistrationInfo info; + + info.plugin_name = PLUGIN_NAME; + info.vendor_name = "Apache Software Foundation"; + info.support_email = "dev@trafficserver.apache.org"; + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSDebug(PLUGIN_NAME, "[%s] Plugin registration failed", PLUGIN_NAME); + + goto Lerror; + } + + /* This is call we could use if we need to protect global data */ + /* TSReleaseAssert ((mutex = TSMutexCreate()) != TS_NULL_MUTEX); */ + + TSMutex mutex = TS_NULL_MUTEX; + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, TSContCreate(global_plugin, mutex)); + TSDebug(PLUGIN_NAME, "[%s] Plugin registration succeeded", PLUGIN_NAME); + return; + +Lerror: + TSDebug(PLUGIN_NAME, "[%s] Plugin disabled", PLUGIN_NAME); +}