From 95751793292f0dac9153771f3324fde9d07ec794 Mon Sep 17 00:00:00 2001 From: bneradt Date: Tue, 3 Dec 2019 16:23:14 +0000 Subject: [PATCH 1/2] Fixing TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK enum value. The value for TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK was out of sync with the corresponding event, TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE. This caused the handler for TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK to never be invoked with the TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE event. This patch fixes this problem and adds comments explaining the otherwise implicit but very important requirement that these values correspond as they do. The naming of the event and hook were off too. A hook with name TS_HTTP_X_HOOK should have an event of name TS_EVENT_HTTP_X, but these got out of sync somehow for request buffer read complete. This adjusts the event name appropriately. --- .../c-api/request_buffer/request_buffer.c | 2 +- include/ts/apidefs.h.in | 289 ++++++++++-------- proxy/http/HttpDebugNames.cc | 4 +- src/traffic_server/InkAPITest.cc | 25 +- .../gold_tests/pluginTest/test_hooks/200.gold | 28 ++ .../pluginTest/test_hooks/body_buffer.test.py | 129 ++++++++ 6 files changed, 340 insertions(+), 137 deletions(-) create mode 100644 tests/gold_tests/pluginTest/test_hooks/200.gold create mode 100644 tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py diff --git a/example/plugins/c-api/request_buffer/request_buffer.c b/example/plugins/c-api/request_buffer/request_buffer.c index 69ab0c6269d..2214cda5243 100644 --- a/example/plugins/c-api/request_buffer/request_buffer.c +++ b/example/plugins/c-api/request_buffer/request_buffer.c @@ -67,7 +67,7 @@ request_buffer_plugin(TSCont contp, TSEvent event, void *edata) { TSDebug(PLUGIN_NAME, "request_buffer_plugin starting, event[%d]", event); TSHttpTxn txnp = (TSHttpTxn)(edata); - if (event == TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE) { + if (event == TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE) { int len = 0; char *body = request_body_get(txnp, &len); TSDebug(PLUGIN_NAME, "request_buffer_plugin gets the request body with length[%d]", len); diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index b2bf28140ce..017fd54179d 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -210,6 +210,122 @@ typedef enum { TS_HTTP_STATUS_NETWORK_AUTHENTICATION_REQUIRED = 511 } TSHttpStatus; +/** + TSEvents are sent to continuations when they are called back. + The TSEvent provides the continuation's handler function with + information about the callback. Based on the event it receives, + the handler function can decide what to do. + + */ +typedef enum { + TS_EVENT_NONE = 0, + TS_EVENT_IMMEDIATE = 1, + TS_EVENT_TIMEOUT = 2, + TS_EVENT_ERROR = 3, + TS_EVENT_CONTINUE = 4, + + TS_EVENT_VCONN_READ_READY = 100, + TS_EVENT_VCONN_WRITE_READY = 101, + TS_EVENT_VCONN_READ_COMPLETE = 102, + TS_EVENT_VCONN_WRITE_COMPLETE = 103, + TS_EVENT_VCONN_EOS = 104, + TS_EVENT_VCONN_INACTIVITY_TIMEOUT = 105, + TS_EVENT_VCONN_ACTIVE_TIMEOUT = 106, + TS_EVENT_VCONN_START = 107, + TS_EVENT_VCONN_CLOSE = 108, + TS_EVENT_VCONN_OUTBOUND_START = 109, + TS_EVENT_VCONN_OUTBOUND_CLOSE = 110, + TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible + + TS_EVENT_NET_CONNECT = 200, + TS_EVENT_NET_CONNECT_FAILED = 201, + TS_EVENT_NET_ACCEPT = 202, + TS_EVENT_NET_ACCEPT_FAILED = 204, + + TS_EVENT_INTERNAL_206 = 206, + TS_EVENT_INTERNAL_207 = 207, + TS_EVENT_INTERNAL_208 = 208, + TS_EVENT_INTERNAL_209 = 209, + TS_EVENT_INTERNAL_210 = 210, + TS_EVENT_INTERNAL_211 = 211, + TS_EVENT_INTERNAL_212 = 212, + + TS_EVENT_HOST_LOOKUP = 500, + + TS_EVENT_CACHE_OPEN_READ = 1102, + TS_EVENT_CACHE_OPEN_READ_FAILED = 1103, + TS_EVENT_CACHE_OPEN_WRITE = 1108, + TS_EVENT_CACHE_OPEN_WRITE_FAILED = 1109, + TS_EVENT_CACHE_REMOVE = 1112, + TS_EVENT_CACHE_REMOVE_FAILED = 1113, + TS_EVENT_CACHE_SCAN = 1120, + TS_EVENT_CACHE_SCAN_FAILED = 1121, + TS_EVENT_CACHE_SCAN_OBJECT = 1122, + TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED = 1123, + TS_EVENT_CACHE_SCAN_OPERATION_FAILED = 1124, + TS_EVENT_CACHE_SCAN_DONE = 1125, + TS_EVENT_CACHE_LOOKUP = 1126, + TS_EVENT_CACHE_READ = 1127, + TS_EVENT_CACHE_DELETE = 1128, + TS_EVENT_CACHE_WRITE = 1129, + TS_EVENT_CACHE_WRITE_HEADER = 1130, + TS_EVENT_CACHE_CLOSE = 1131, + TS_EVENT_CACHE_LOOKUP_READY = 1132, + TS_EVENT_CACHE_LOOKUP_COMPLETE = 1133, + TS_EVENT_CACHE_READ_READY = 1134, + TS_EVENT_CACHE_READ_COMPLETE = 1135, + + TS_EVENT_INTERNAL_1200 = 1200, + + TS_EVENT_SSL_SESSION_GET = 2000, + TS_EVENT_SSL_SESSION_NEW = 2001, + TS_EVENT_SSL_SESSION_REMOVE = 2002, + + TS_EVENT_AIO_DONE = 3900, + + TS_EVENT_HTTP_CONTINUE = 60000, + TS_EVENT_HTTP_ERROR = 60001, + TS_EVENT_HTTP_READ_REQUEST_HDR = 60002, + TS_EVENT_HTTP_OS_DNS = 60003, + TS_EVENT_HTTP_SEND_REQUEST_HDR = 60004, + TS_EVENT_HTTP_READ_CACHE_HDR = 60005, + TS_EVENT_HTTP_READ_RESPONSE_HDR = 60006, + TS_EVENT_HTTP_SEND_RESPONSE_HDR = 60007, + TS_EVENT_HTTP_REQUEST_TRANSFORM = 60008, + TS_EVENT_HTTP_RESPONSE_TRANSFORM = 60009, + TS_EVENT_HTTP_SELECT_ALT = 60010, + TS_EVENT_HTTP_TXN_START = 60011, + TS_EVENT_HTTP_TXN_CLOSE = 60012, + TS_EVENT_HTTP_SSN_START = 60013, + TS_EVENT_HTTP_SSN_CLOSE = 60014, + TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, + TS_EVENT_HTTP_PRE_REMAP = 60016, + TS_EVENT_HTTP_POST_REMAP = 60017, + TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE = 60018, + TS_EVENT_HTTP_RESPONSE_CLIENT = 60019, + + TS_EVENT_LIFECYCLE_PORTS_INITIALIZED = 60100, + TS_EVENT_LIFECYCLE_PORTS_READY = 60101, + TS_EVENT_LIFECYCLE_CACHE_READY = 60102, + TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED = 60103, + TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60104, + TS_EVENT_LIFECYCLE_MSG = 60105, + TS_EVENT_LIFECYCLE_TASK_THREADS_READY = 60106, + TS_EVENT_LIFECYCLE_SHUTDOWN = 60107, + + TS_EVENT_INTERNAL_60200 = 60200, + TS_EVENT_INTERNAL_60201 = 60201, + TS_EVENT_INTERNAL_60202 = 60202, + TS_EVENT_SSL_CERT = 60203, + TS_EVENT_SSL_SERVERNAME = 60204, + TS_EVENT_SSL_VERIFY_SERVER = 60205, + TS_EVENT_SSL_VERIFY_CLIENT = 60206, + TS_EVENT_SSL_CLIENT_HELLO = 60207, + + TS_EVENT_MGMT_UPDATE = 60300 +} TSEvent; +#define TS_EVENT_HTTP_READ_REQUEST_PRE_REMAP TS_EVENT_HTTP_PRE_REMAP /* backwards compat */ + /** This set of enums represents the possible hooks where you can set up continuation callbacks. The functions used to register a @@ -262,25 +378,48 @@ typedef enum { TS_HTTP_LAST_HOOK _must_ be the last element. Only right place to insert a new element is just before TS_HTTP_LAST_HOOK. + @note The TS_HTTP hooks below have to be in the same order as their + corresponding TS_EVENT counterparts. We use this in calculating the + corresponding event from a hook ID by summing + TS_EVENT_HTTP_READ_REQUEST_HDR with the hook ID (see the invoke call in + HttpSM::state_api_callout). For example, the following expression must + be true: + + TS_EVENT_HTTP_TXN_CLOSE == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_TXN_CLOSE_HOOK + */ typedef enum { - TS_HTTP_READ_REQUEST_HDR_HOOK, - TS_HTTP_OS_DNS_HOOK, - TS_HTTP_SEND_REQUEST_HDR_HOOK, - TS_HTTP_READ_CACHE_HDR_HOOK, - TS_HTTP_READ_RESPONSE_HDR_HOOK, - TS_HTTP_SEND_RESPONSE_HDR_HOOK, - TS_HTTP_REQUEST_TRANSFORM_HOOK, - TS_HTTP_RESPONSE_TRANSFORM_HOOK, - TS_HTTP_SELECT_ALT_HOOK, - TS_HTTP_TXN_START_HOOK, - TS_HTTP_TXN_CLOSE_HOOK, - TS_HTTP_SSN_START_HOOK, - TS_HTTP_SSN_CLOSE_HOOK, - TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, - TS_HTTP_PRE_REMAP_HOOK, - TS_HTTP_POST_REMAP_HOOK, - TS_HTTP_RESPONSE_CLIENT_HOOK, +/* + The following macro helps maintain the relationship between the TS_HTTP + hooks and their TS_EVENT_HTTP counterparts as described in the above + doxygen comment. +*/ +#define REBASE(COMMON) TS_HTTP_##COMMON##_HOOK = TS_EVENT_HTTP_##COMMON - TS_EVENT_HTTP_READ_REQUEST_HDR + REBASE(READ_REQUEST_HDR), // TS_HTTP_READ_REQUEST_HDR_HOOK + REBASE(OS_DNS), // TS_HTTP_OS_DNS_HOOK + REBASE(SEND_REQUEST_HDR), // TS_HTTP_SEND_REQUEST_HDR_HOOK + REBASE(READ_CACHE_HDR), // TS_HTTP_READ_CACHE_HDR_HOOK + REBASE(READ_RESPONSE_HDR), // TS_HTTP_READ_RESPONSE_HDR_HOOK + REBASE(SEND_RESPONSE_HDR), // TS_HTTP_SEND_RESPONSE_HDR_HOOK + REBASE(REQUEST_TRANSFORM), // TS_HTTP_REQUEST_TRANSFORM_HOOK + REBASE(RESPONSE_TRANSFORM), // TS_HTTP_RESPONSE_TRANSFORM_HOOK + REBASE(SELECT_ALT), // TS_HTTP_SELECT_ALT_HOOK + REBASE(TXN_START), // TS_HTTP_TXN_START_HOOK + REBASE(TXN_CLOSE), // TS_HTTP_TXN_CLOSE_HOOK + REBASE(SSN_START), // TS_HTTP_SSN_START_HOOK + REBASE(SSN_CLOSE), // TS_HTTP_SSN_CLOSE_HOOK + REBASE(CACHE_LOOKUP_COMPLETE), // TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK + REBASE(PRE_REMAP), // TS_HTTP_PRE_REMAP_HOOK + REBASE(POST_REMAP), // TS_HTTP_POST_REMAP_HOOK + REBASE(REQUEST_BUFFER_READ_COMPLETE), // TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK + REBASE(RESPONSE_CLIENT), // TS_HTTP_RESPONSE_CLIENT_HOOK +#undef REBASE + + // NOTE: + // If adding any TS_HTTP hooks, be sure to understand the note above in the + // doxygen comment about the ordering of these enum values with respect to + // their corresponding TS_EVENT values. + // Putting the SSL hooks in the same enum space // So both sets of hooks can be set by the same Hook function TS_SSL_FIRST_HOOK, @@ -298,7 +437,6 @@ typedef enum { TS_VCONN_OUTBOUND_START_HOOK, TS_VCONN_OUTBOUND_CLOSE_HOOK, TS_SSL_LAST_HOOK = TS_VCONN_OUTBOUND_CLOSE_HOOK, - TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, TS_HTTP_LAST_HOOK } TSHttpHookID; @@ -387,121 +525,6 @@ typedef enum { TS_LIFECYCLE_LAST_HOOK } TSLifecycleHookID; -/** - TSEvents are sent to continuations when they are called back. - The TSEvent provides the continuation's handler function with - information about the callback. Based on the event it receives, - the handler function can decide what to do. - - */ -typedef enum { - TS_EVENT_NONE = 0, - TS_EVENT_IMMEDIATE = 1, - TS_EVENT_TIMEOUT = 2, - TS_EVENT_ERROR = 3, - TS_EVENT_CONTINUE = 4, - - TS_EVENT_VCONN_READ_READY = 100, - TS_EVENT_VCONN_WRITE_READY = 101, - TS_EVENT_VCONN_READ_COMPLETE = 102, - TS_EVENT_VCONN_WRITE_COMPLETE = 103, - TS_EVENT_VCONN_EOS = 104, - TS_EVENT_VCONN_INACTIVITY_TIMEOUT = 105, - TS_EVENT_VCONN_ACTIVE_TIMEOUT = 106, - TS_EVENT_VCONN_START = 107, - TS_EVENT_VCONN_CLOSE = 108, - TS_EVENT_VCONN_OUTBOUND_START = 109, - TS_EVENT_VCONN_OUTBOUND_CLOSE = 110, - TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible - - TS_EVENT_NET_CONNECT = 200, - TS_EVENT_NET_CONNECT_FAILED = 201, - TS_EVENT_NET_ACCEPT = 202, - TS_EVENT_NET_ACCEPT_FAILED = 204, - - TS_EVENT_INTERNAL_206 = 206, - TS_EVENT_INTERNAL_207 = 207, - TS_EVENT_INTERNAL_208 = 208, - TS_EVENT_INTERNAL_209 = 209, - TS_EVENT_INTERNAL_210 = 210, - TS_EVENT_INTERNAL_211 = 211, - TS_EVENT_INTERNAL_212 = 212, - - TS_EVENT_HOST_LOOKUP = 500, - - TS_EVENT_CACHE_OPEN_READ = 1102, - TS_EVENT_CACHE_OPEN_READ_FAILED = 1103, - TS_EVENT_CACHE_OPEN_WRITE = 1108, - TS_EVENT_CACHE_OPEN_WRITE_FAILED = 1109, - TS_EVENT_CACHE_REMOVE = 1112, - TS_EVENT_CACHE_REMOVE_FAILED = 1113, - TS_EVENT_CACHE_SCAN = 1120, - TS_EVENT_CACHE_SCAN_FAILED = 1121, - TS_EVENT_CACHE_SCAN_OBJECT = 1122, - TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED = 1123, - TS_EVENT_CACHE_SCAN_OPERATION_FAILED = 1124, - TS_EVENT_CACHE_SCAN_DONE = 1125, - TS_EVENT_CACHE_LOOKUP = 1126, - TS_EVENT_CACHE_READ = 1127, - TS_EVENT_CACHE_DELETE = 1128, - TS_EVENT_CACHE_WRITE = 1129, - TS_EVENT_CACHE_WRITE_HEADER = 1130, - TS_EVENT_CACHE_CLOSE = 1131, - TS_EVENT_CACHE_LOOKUP_READY = 1132, - TS_EVENT_CACHE_LOOKUP_COMPLETE = 1133, - TS_EVENT_CACHE_READ_READY = 1134, - TS_EVENT_CACHE_READ_COMPLETE = 1135, - - TS_EVENT_INTERNAL_1200 = 1200, - - TS_EVENT_SSL_SESSION_GET = 2000, - TS_EVENT_SSL_SESSION_NEW = 2001, - TS_EVENT_SSL_SESSION_REMOVE = 2002, - - TS_EVENT_AIO_DONE = 3900, - - TS_EVENT_HTTP_CONTINUE = 60000, - TS_EVENT_HTTP_ERROR = 60001, - TS_EVENT_HTTP_READ_REQUEST_HDR = 60002, - TS_EVENT_HTTP_OS_DNS = 60003, - TS_EVENT_HTTP_SEND_REQUEST_HDR = 60004, - TS_EVENT_HTTP_READ_CACHE_HDR = 60005, - TS_EVENT_HTTP_READ_RESPONSE_HDR = 60006, - TS_EVENT_HTTP_SEND_RESPONSE_HDR = 60007, - TS_EVENT_HTTP_REQUEST_TRANSFORM = 60008, - TS_EVENT_HTTP_RESPONSE_TRANSFORM = 60009, - TS_EVENT_HTTP_SELECT_ALT = 60010, - TS_EVENT_HTTP_TXN_START = 60011, - TS_EVENT_HTTP_TXN_CLOSE = 60012, - TS_EVENT_HTTP_SSN_START = 60013, - TS_EVENT_HTTP_SSN_CLOSE = 60014, - TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, - TS_EVENT_HTTP_PRE_REMAP = 60016, - TS_EVENT_HTTP_POST_REMAP = 60017, - TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE = 60018, - - TS_EVENT_LIFECYCLE_PORTS_INITIALIZED = 60100, - TS_EVENT_LIFECYCLE_PORTS_READY = 60101, - TS_EVENT_LIFECYCLE_CACHE_READY = 60102, - TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED = 60103, - TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60104, - TS_EVENT_LIFECYCLE_MSG = 60105, - TS_EVENT_LIFECYCLE_TASK_THREADS_READY = 60106, - TS_EVENT_LIFECYCLE_SHUTDOWN = 60107, - - TS_EVENT_INTERNAL_60200 = 60200, - TS_EVENT_INTERNAL_60201 = 60201, - TS_EVENT_INTERNAL_60202 = 60202, - TS_EVENT_SSL_CERT = 60203, - TS_EVENT_SSL_SERVERNAME = 60204, - TS_EVENT_SSL_VERIFY_SERVER = 60205, - TS_EVENT_SSL_VERIFY_CLIENT = 60206, - TS_EVENT_SSL_CLIENT_HELLO = 60207, - - TS_EVENT_MGMT_UPDATE = 60300 -} TSEvent; -#define TS_EVENT_HTTP_READ_REQUEST_PRE_REMAP TS_EVENT_HTTP_PRE_REMAP /* backwards compat */ - typedef enum { TS_SRVSTATE_STATE_UNDEFINED = 0, TS_SRVSTATE_ACTIVE_TIMEOUT, diff --git a/proxy/http/HttpDebugNames.cc b/proxy/http/HttpDebugNames.cc index f99e5b1c135..3f691e6546c 100644 --- a/proxy/http/HttpDebugNames.cc +++ b/proxy/http/HttpDebugNames.cc @@ -355,8 +355,8 @@ HttpDebugNames::get_event_name(int event) return "TS_EVENT_VCONN_CLOSE"; case TS_EVENT_LIFECYCLE_MSG: return "TS_EVENT_LIFECYCLE_MSG"; - case TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE: - return "TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE"; + case TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE: + return "TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE"; case TS_EVENT_MGMT_UPDATE: return "TS_EVENT_MGMT_UPDATE"; case TS_EVENT_INTERNAL_60200: diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index 2b137453b92..a2dd0f2ba2c 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -83,6 +83,29 @@ #define ERROR_BODY "TESTING ERROR PAGE" #define TRANSFORM_APPEND_STRING "This is a transformed response" +////////////////////////////////////////////////////////////////////////////// +// These static asserts verify the hook vs event value relationship described +// in the doxygen comment for TSHttpHookID. +////////////////////////////////////////////////////////////////////////////// +static_assert(TS_EVENT_HTTP_READ_REQUEST_HDR == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_READ_REQUEST_HDR_HOOK); +static_assert(TS_EVENT_HTTP_OS_DNS == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_OS_DNS_HOOK); +static_assert(TS_EVENT_HTTP_SEND_REQUEST_HDR == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_SEND_REQUEST_HDR_HOOK); +static_assert(TS_EVENT_HTTP_READ_CACHE_HDR == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_READ_CACHE_HDR_HOOK); +static_assert(TS_EVENT_HTTP_READ_RESPONSE_HDR == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_READ_RESPONSE_HDR_HOOK); +static_assert(TS_EVENT_HTTP_SEND_RESPONSE_HDR == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_SEND_RESPONSE_HDR_HOOK); +static_assert(TS_EVENT_HTTP_REQUEST_TRANSFORM == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_REQUEST_TRANSFORM_HOOK); +static_assert(TS_EVENT_HTTP_RESPONSE_TRANSFORM == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_RESPONSE_TRANSFORM_HOOK); +static_assert(TS_EVENT_HTTP_SELECT_ALT == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_SELECT_ALT_HOOK); +static_assert(TS_EVENT_HTTP_TXN_START == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_TXN_START_HOOK); +static_assert(TS_EVENT_HTTP_TXN_CLOSE == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_TXN_CLOSE_HOOK); +static_assert(TS_EVENT_HTTP_SSN_START == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_SSN_START_HOOK); +static_assert(TS_EVENT_HTTP_SSN_CLOSE == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_SSN_CLOSE_HOOK); +static_assert(TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK); +static_assert(TS_EVENT_HTTP_PRE_REMAP == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_PRE_REMAP_HOOK); +static_assert(TS_EVENT_HTTP_POST_REMAP == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_POST_REMAP_HOOK); +static_assert(TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE == + TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK); + ////////////////////////////////////////////////////////////////////////////// // STRUCTURES ////////////////////////////////////////////////////////////////////////////// @@ -6603,6 +6626,7 @@ enum ORIG_TSHttpHookID { ORIG_TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, ORIG_TS_HTTP_PRE_REMAP_HOOK, ORIG_TS_HTTP_POST_REMAP_HOOK, + ORIG_TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, ORIG_TS_HTTP_RESPONSE_CLIENT_HOOK, ORIG_TS_SSL_FIRST_HOOK, ORIG_TS_VCONN_START_HOOK = ORIG_TS_SSL_FIRST_HOOK, @@ -6616,7 +6640,6 @@ enum ORIG_TSHttpHookID { ORIG_TS_VCONN_OUTBOUND_START_HOOK, ORIG_TS_VCONN_OUTBOUND_CLOSE_HOOK, ORIG_TS_SSL_LAST_HOOK = ORIG_TS_VCONN_OUTBOUND_CLOSE_HOOK, - ORIG_TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, ORIG_TS_HTTP_LAST_HOOK }; diff --git a/tests/gold_tests/pluginTest/test_hooks/200.gold b/tests/gold_tests/pluginTest/test_hooks/200.gold new file mode 100644 index 00000000000..a8875735ebb --- /dev/null +++ b/tests/gold_tests/pluginTest/test_hooks/200.gold @@ -0,0 +1,28 @@ +`` +> POST /contentlength HTTP/1.1`` +> Host:`` +> User-Agent: curl/`` +> Accept: */*`` +> Content-Length: 22`` +> Content-Type: application/`` +`` +< HTTP/1.1 200 OK`` +< Server: ATS/`` +< Content-Length: 23`` +< Date:`` +< Age:`` +`` +> POST /chunked`` +> Host:`` +> User-Agent: curl/`` +> Accept: */*`` +> Transfer-Encoding: chunked`` +> Content-Type: application/`` +`` +< HTTP/1.1 200 OK`` +< Server: ATS/`` +< Date: `` +< Age: 0`` +< Transfer-Encoding: chunked`` +< Connection: keep-alive`` +`` diff --git a/tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py b/tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py new file mode 100644 index 00000000000..7e84e8914a4 --- /dev/null +++ b/tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py @@ -0,0 +1,129 @@ +''' +Verify HTTP body buffering. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + + +def int_to_hex_string(int_value): + ''' + Convert the given int value to a hex string with no '0x' prefix. + + >>> int_to_hex_string(0) + '0' + >>> int_to_hex_string(1) + '1' + >>> int_to_hex_string(10) + 'r' + >>> int_to_hex_string(16) + '0' + >>> int_to_hex_string(17) + 'f1' + ''' + if type(int_value) != int: + raise ValueError("Input should be an int type.") + return hex(int_value).split('x')[1] + + +class BodyBufferTest: + def __init__(cls, description): + Test.Summary = description + cls._origin_max_connections = 3 + cls.setupOriginServer() + cls.setupTS() + + def setupOriginServer(self): + self._server = Test.MakeOriginServer("server") + self.content_length_request_body = "content-length request" + request_header = {"headers": "POST /contentlength HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Content-Length: {}\r\n\r\n".format( + len(self.content_length_request_body)), + "timestamp": "1469733493.993", + "body": self.content_length_request_body} + content_length_response_body = "content-length response" + response_header = {"headers": "HTTP/1.1 200 OK\r\n" + "Server: microserver\r\n" + "Content-Length: {}\r\n\r\n" + "Connection: close\r\n\r\n".format( + len(content_length_response_body)), + "timestamp": "1469733493.993", + "body": content_length_response_body} + self._server.addResponse("sessionlog.json", request_header, response_header) + + self.chunked_request_body = "chunked request" + self.encoded_chunked_request = "{0}\r\n{1}\r\n0\r\n\r\n".format( + int_to_hex_string(len(self.chunked_request_body)), self.chunked_request_body) + request_header2 = {"headers": "POST /chunked HTTP/1.1\r\n" + "Transfer-Encoding: chunked\r\n" + "Host: www.example.com\r\n" + "Connection: keep-alive\r\n\r\n", + "timestamp": "1469733493.993", + "body": self.encoded_chunked_request} + self.chunked_response_body = "chunked response" + self.encoded_chunked_response = "{0}\r\n{1}\r\n0\r\n\r\n".format( + int_to_hex_string(len(self.chunked_response_body)), self.chunked_response_body) + response_header2 = {"headers": "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "Server: microserver\r\n" + "Connection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": self.encoded_chunked_response} + self._server.addResponse("sessionlog.json", request_header2, response_header2) + + def setupTS(self): + self._ts = Test.MakeATSProcess("ts", select_ports=False) + self._ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(self._server.Variables.Port) + ) + Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'request_buffer.c'), self._ts) + self._ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'request_buffer', + 'proxy.config.http.per_server.connection.max': self._origin_max_connections, + # Disable queueing when connection reaches limit + 'proxy.config.http.per_server.connection.queue_size': 0, + }) + + self._ts.Streams.stderr = Testers.ContainsExpression( + "request_buffer_plugin gets the request body with length\[{0}\]: \[{1}\]".format( + len(self.content_length_request_body), self.content_length_request_body), + "Verify that the plugin parsed the content-length request body data.") + self._ts.Streams.stderr += Testers.ContainsExpression( + "request_buffer_plugin gets the request body with length\[{0}\]: \[".format( + len(self.encoded_chunked_request)), + "Verify that the plugin parsed the chunked request body.") + self._ts.Streams.stderr += Testers.ContainsExpression( + "^{}".format(self.chunked_request_body), + "Verify that the plugin parsed the chunked request body data.") + + def run(self): + tr = Test.AddTestRun() + # Send both a Content-Length request and a chunked-encoded request. + tr.Processes.Default.Command = ( + 'curl -v http://127.0.0.1:{0}/contentlength -d "{1}" ; ' + 'curl -v http://127.0.0.1:{0}/chunked -H "Transfer-Encoding: chunked" -d "{2}"'.format( + self._ts.Variables.port, self.content_length_request_body, self.chunked_request_body)) + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(Test.Processes.ts) + tr.Processes.Default.Streams.stderr = "200.gold" + + +bodyBufferTest = BodyBufferTest("Test request body buffering.") +bodyBufferTest.run() From b34a8ab1f06f206accdf0a31908abdd5dd952aea Mon Sep 17 00:00:00 2001 From: bneradt Date: Thu, 12 Dec 2019 23:12:21 +0000 Subject: [PATCH 2/2] traffic_dump: print request body data and client address filtering This adds the -b option to traffic dump to dump client request body data. It also adds the -4 and -6 options so that the client can filter what is dumped based upon a client IP address. --- include/tscore/ink_inet.h | 1 + .../experimental/traffic_dump/json_utils.cc | 41 ++-- .../experimental/traffic_dump/json_utils.h | 14 ++ .../experimental/traffic_dump/session_data.cc | 51 ++++- .../experimental/traffic_dump/session_data.h | 25 ++- .../experimental/traffic_dump/traffic_dump.cc | 37 +++- .../traffic_dump/transaction_data.cc | 80 +++++++- .../traffic_dump/transaction_data.h | 52 ++++- proxy/http/HttpSM.cc | 22 ++- .../pluginTest/test_hooks/body_buffer.test.py | 29 ++- .../pluginTest/traffic_dump/gold/200_get.gold | 12 ++ ...{200.gold => 200_get_sensitive_field.gold} | 0 .../traffic_dump/gold/200_post.gold | 12 ++ .../traffic_dump/traffic_dump.test.py | 8 +- .../traffic_dump_ip_filter.test.py | 150 +++++++++++++++ .../traffic_dump_request_body.test.py | 182 ++++++++++++++++++ .../pluginTest/traffic_dump/verify_replay.py | 39 +++- 17 files changed, 678 insertions(+), 77 deletions(-) create mode 100644 tests/gold_tests/pluginTest/traffic_dump/gold/200_get.gold rename tests/gold_tests/pluginTest/traffic_dump/gold/{200.gold => 200_get_sensitive_field.gold} (100%) create mode 100644 tests/gold_tests/pluginTest/traffic_dump/gold/200_post.gold create mode 100644 tests/gold_tests/pluginTest/traffic_dump/traffic_dump_ip_filter.test.py create mode 100644 tests/gold_tests/pluginTest/traffic_dump/traffic_dump_request_body.test.py diff --git a/include/tscore/ink_inet.h b/include/tscore/ink_inet.h index aa6ddf057dc..63310daa5ff 100644 --- a/include/tscore/ink_inet.h +++ b/include/tscore/ink_inet.h @@ -1179,6 +1179,7 @@ struct IpAddr { /// Construct from @c sockaddr. explicit IpAddr(sockaddr const *addr) { this->assign(addr); } + explicit IpAddr(sockaddr const &addr) { this->assign(&addr); } /// Construct from @c sockaddr_in6. explicit IpAddr(sockaddr_in6 const &addr) { this->assign(ats_ip_sa_cast(&addr)); } /// Construct from @c sockaddr_in6. diff --git a/plugins/experimental/traffic_dump/json_utils.cc b/plugins/experimental/traffic_dump/json_utils.cc index 3b5fbbdc690..e45f9c753f4 100644 --- a/plugins/experimental/traffic_dump/json_utils.cc +++ b/plugins/experimental/traffic_dump/json_utils.cc @@ -132,31 +132,6 @@ esc_json_out(const char *buf, int64_t len, std::ostream &jsonfile) return len; } -/** Escape characters in a string as needed and return the resultant escaped string. - * - * @param[in] s The characters that need to be escaped. - */ -std::string -escape_json(std::string_view s) -{ - std::ostringstream o; - esc_json_out(s.data(), s.length(), o); - return o.str(); -} - -/** An escape_json overload for a char buffer. - * - * @param[in] buf The char buffer pointer with characters that need to be escaped. - * - * @param[in] size The size of the buf char array. - */ -std::string -escape_json(char const *buf, int64_t size) -{ - std::ostringstream o; - esc_json_out(buf, size, o); - return o.str(); -} } // anonymous namespace @@ -180,4 +155,20 @@ json_entry_array(std::string_view name, std::string_view value) return "[\"" + escape_json(name) + "\",\"" + escape_json(value) + "\"]"; } +std::string +escape_json(std::string_view s) +{ + std::ostringstream o; + esc_json_out(s.data(), s.length(), o); + return o.str(); +} + +std::string +escape_json(char const *buf, int64_t size) +{ + std::ostringstream o; + esc_json_out(buf, size, o); + return o.str(); +} + } // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/json_utils.h b/plugins/experimental/traffic_dump/json_utils.h index e91829cb45a..839ea081de7 100644 --- a/plugins/experimental/traffic_dump/json_utils.h +++ b/plugins/experimental/traffic_dump/json_utils.h @@ -60,4 +60,18 @@ std::string json_entry(std::string_view name, char const *value, int64_t size); */ std::string json_entry_array(std::string_view name, std::string_view value); +/** Escape characters in a string as needed and return the resultant escaped string. + * + * @param[in] s The characters that need to be escaped. + */ +std::string escape_json(std::string_view s); + +/** An escape_json overload for a char buffer. + * + * @param[in] buf The char buffer pointer with characters that need to be escaped. + * + * @param[in] size The size of the buf char array. + */ +std::string escape_json(char const *buf, int64_t size); + } // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/session_data.cc b/plugins/experimental/traffic_dump/session_data.cc index de41e09dc6a..5572a4ba009 100644 --- a/plugins/experimental/traffic_dump/session_data.cc +++ b/plugins/experimental/traffic_dump/session_data.cc @@ -196,6 +196,7 @@ std::atomic SessionData::disk_usage = 0; ts::file::path SessionData::log_directory{default_log_directory}; uint64_t SessionData::session_counter = 0; std::string SessionData::sni_filter; +std::optional SessionData::client_ip_filter = std::nullopt; int SessionData::get_session_arg_index() @@ -222,12 +223,25 @@ SessionData::set_max_disk_usage(int64_t new_max_disk_usage) } bool -SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size) +SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view ip_filter) { SessionData::log_directory = log_directory; SessionData::max_disk_usage = max_disk_usage; SessionData::sample_pool_size = sample_size; + if (!ip_filter.empty()) { + client_ip_filter.emplace(); + if (client_ip_filter->load(ip_filter)) { + TSDebug(debug_tag, "Problems parsing IP filter address argument: %.*s", static_cast(ip_filter.size()), ip_filter.data()); + TSError("[%s] Problems parsing IP filter address argument: %.*s", debug_tag, static_cast(ip_filter.size()), + ip_filter.data()); + client_ip_filter = std::nullopt; + return false; + } else { + TSDebug(debug_tag, "Filtering to only dump connections with ip: %.*s", static_cast(ip_filter.size()), ip_filter.data()); + } + } + if (TS_SUCCESS != TSUserArgIndexReserve(TS_USER_ARGS_SSN, debug_tag, "Track log related data", &session_arg_index)) { TSError("[%s] Unable to initialize plugin (disabled). Failed to reserve ssn arg.", traffic_dump::debug_tag); return false; @@ -244,9 +258,10 @@ SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_ } bool -SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view sni_filter) +SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view ip_filter, + std::string_view sni_filter) { - if (!init(log_directory, max_disk_usage, sample_size)) { + if (!init(log_directory, max_disk_usage, sample_size, ip_filter)) { return false; } SessionData::sni_filter = sni_filter; @@ -342,6 +357,25 @@ SessionData::write_transaction_to_disk(std::string_view content) return result; } +bool +SessionData::is_filtered_out(const sockaddr *session_client_ip) +{ + if (!client_ip_filter) { + // The user did not configure an IP by which to filter. + return false; + } + if (session_client_ip == nullptr) { + TSDebug(debug_tag, "Found no client IP address for session. Abort."); + return true; + } + if (session_client_ip->sa_family != AF_INET && session_client_ip->sa_family != AF_INET6) { + TSDebug(debug_tag, "IP family is not v4 nor v6. Abort."); + return true; + } + + IpAddr session_address(*session_client_ip); + return session_address != *client_ip_filter; +} int SessionData::session_aio_handler(TSCont contp, TSEvent event, void *edata) { @@ -418,12 +452,17 @@ SessionData::global_session_handler(TSCont contp, TSEvent event, void *edata) } const auto this_session_count = session_counter++; if (this_session_count % sample_pool_size != 0) { - TSDebug(debug_tag, "global_session_handler(): Ignore session %" PRId64 "...", id); + TSDebug(debug_tag, "Ignore session %" PRId64 " per the random sampling mechanism", id); break; } else if (disk_usage >= max_disk_usage) { - TSDebug(debug_tag, "global_session_handler(): Ignore session %" PRId64 "due to disk usage %" PRId64 "bytes", id, - disk_usage.load()); + TSDebug(debug_tag, "Ignore session %" PRId64 "due to disk usage %" PRId64 "bytes", id, disk_usage.load()); break; + } else { + const sockaddr *client_ip = TSHttpSsnClientAddrGet(ssnp); + if (SessionData::is_filtered_out(client_ip)) { + TSDebug(debug_tag, "Ignore session %" PRId64 " per the client's IP filter", id); + break; + } } // Beginning of a new session /// Get epoch time diff --git a/plugins/experimental/traffic_dump/session_data.h b/plugins/experimental/traffic_dump/session_data.h index b25a42f57bb..0cac9b0e751 100644 --- a/plugins/experimental/traffic_dump/session_data.h +++ b/plugins/experimental/traffic_dump/session_data.h @@ -29,6 +29,7 @@ #include #include "ts/ts.h" +#include "tscore/ink_inet.h" #include "tscore/ts_file.h" namespace traffic_dump @@ -100,6 +101,9 @@ class SessionData /// The running counter of all sessions dumped by traffic_dump. static uint64_t session_counter; + /// Only addresses with this IP will be dumped (if set). + static std::optional client_ip_filter; + public: SessionData(); ~SessionData(); @@ -111,8 +115,9 @@ class SessionData * * @return True if initialization is successful, false otherwise. */ - static bool init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size); - static bool init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view sni_filter); + static bool init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view ip_filter); + static bool init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view ip_filter, + std::string_view sni_filter); /** Set the sample_pool_size to a new value. * @@ -173,6 +178,22 @@ class SessionData */ int write_to_disk_no_lock(std::string_view content); + /** Determine whether the user configured IP filter indicates this transaction + * should not be dumped. + * + * @note This also does some validity verification on the IP, making sure it is + * v4 or v6, and returns true (i.e., it should be filtered out) if it does not + * match these checks. These checks are required in order to perform the IP + * filtering logic. This function will only return true if the client has + * enabled client filtering. + * + * @param[in] session_client_ip The session's client address. + * + * @return True if the provided address should not be dumped per the user's + * configuration, false otherwise. + */ + static bool is_filtered_out(const sockaddr *session_client_ip); + /** Get the JSON string that describes the client session stack. * * @param[in] ssnp The reference to the client session. diff --git a/plugins/experimental/traffic_dump/traffic_dump.cc b/plugins/experimental/traffic_dump/traffic_dump.cc index 38ae1f17ba5..115ebeeca69 100644 --- a/plugins/experimental/traffic_dump/traffic_dump.cc +++ b/plugins/experimental/traffic_dump/traffic_dump.cc @@ -81,22 +81,33 @@ TSPluginInit(int argc, char const *argv[]) return; } + bool dump_body = false; bool sensitive_fields_were_specified = false; traffic_dump::sensitive_fields_t user_specified_fields; ts::file::path log_dir{traffic_dump::SessionData::default_log_directory}; int64_t sample_pool_size = traffic_dump::SessionData::default_sample_pool_size; int64_t max_disk_usage = traffic_dump::SessionData::default_max_disk_usage; std::string sni_filter; + std::string client_ip_filter; /// Commandline options - static const struct option longopts[] = { - {"logdir", required_argument, nullptr, 'l'}, {"sample", required_argument, nullptr, 's'}, - {"limit", required_argument, nullptr, 'm'}, {"sensitive-fields", required_argument, nullptr, 'f'}, - {"sni-filter", required_argument, nullptr, 'n'}, {nullptr, no_argument, nullptr, 0}}; - int opt = 0; + static const struct option longopts[] = {{"dump_body", no_argument, nullptr, 'b'}, + {"logdir", required_argument, nullptr, 'l'}, + {"sample", required_argument, nullptr, 's'}, + {"limit", required_argument, nullptr, 'm'}, + {"sensitive-fields", required_argument, nullptr, 'f'}, + {"sni-filter", required_argument, nullptr, 'n'}, + {"client_ipv4", required_argument, nullptr, '4'}, + {"client_ipv6", required_argument, nullptr, '6'}, + {nullptr, no_argument, nullptr, 0}}; + int opt = 0; while (opt >= 0) { - opt = getopt_long(argc, const_cast(argv), "l:", longopts, nullptr); + opt = getopt_long(argc, const_cast(argv), "bf:l:s:m:n:4:6", longopts, nullptr); switch (opt) { + case 'b': { + dump_body = true; + break; + } case 'f': { // --sensitive-fields takes a comma-separated list of HTTP fields that // are sensitive. The field values for these fields will be replaced @@ -133,6 +144,12 @@ TSPluginInit(int argc, char const *argv[]) } case 'm': { max_disk_usage = static_cast(std::strtol(optarg, nullptr, 0)); + break; + } + case '4': + case '6': { + client_ip_filter = std::string(optarg); + break; } case -1: case '?': @@ -148,26 +165,26 @@ TSPluginInit(int argc, char const *argv[]) log_dir = ts::file::path(TSInstallDirGet()) / log_dir; } if (sni_filter.empty()) { - if (!traffic_dump::SessionData::init(log_dir.view(), max_disk_usage, sample_pool_size)) { + if (!traffic_dump::SessionData::init(log_dir.view(), max_disk_usage, sample_pool_size, client_ip_filter)) { TSError("[%s] Failed to initialize session state.", traffic_dump::debug_tag); return; } } else { - if (!traffic_dump::SessionData::init(log_dir.view(), max_disk_usage, sample_pool_size, sni_filter)) { + if (!traffic_dump::SessionData::init(log_dir.view(), max_disk_usage, sample_pool_size, client_ip_filter, sni_filter)) { TSError("[%s] Failed to initialize session state with an SNI filter.", traffic_dump::debug_tag); return; } } if (sensitive_fields_were_specified) { - if (!traffic_dump::TransactionData::init(std::move(user_specified_fields))) { + if (!traffic_dump::TransactionData::init(dump_body, std::move(user_specified_fields))) { TSError("[%s] Failed to initialize transaction state with user-specified fields.", traffic_dump::debug_tag); return; } } else { // The user did not provide their own list of sensitive fields. Use the // default. - if (!traffic_dump::TransactionData::init()) { + if (!traffic_dump::TransactionData::init(dump_body)) { TSError("[%s] Failed to initialize transaction state.", traffic_dump::debug_tag); return; } diff --git a/plugins/experimental/traffic_dump/transaction_data.cc b/plugins/experimental/traffic_dump/transaction_data.cc index 51f63507db4..1b2b96a59aa 100644 --- a/plugins/experimental/traffic_dump/transaction_data.cc +++ b/plugins/experimental/traffic_dump/transaction_data.cc @@ -21,15 +21,18 @@ limitations under the License. */ -#include "transaction_data.h" +#include + #include "global_variables.h" #include "json_utils.h" #include "session_data.h" +#include "transaction_data.h" namespace traffic_dump { int TransactionData::transaction_arg_index = 0; sensitive_fields_t TransactionData::sensitive_fields; +bool TransactionData::dump_body = false; std::string default_sensitive_field_value; @@ -44,6 +47,47 @@ sensitive_fields_t default_sensitive_fields = { "Cookie", }; +std::string +TransactionData::request_body_get(TSHttpTxn txnp) +{ + TSDebug(debug_tag, "Getting the HTTP body"); + TSIOBufferReader post_buffer_reader = TSHttpTxnPostBufferReaderGet(txnp); + int64_t read_avail = TSIOBufferReaderAvail(post_buffer_reader); + if (read_avail == 0) { + TSIOBufferReaderFree(post_buffer_reader); + return {}; + } + + // std::string has no reserving allocator. So create an explicitly empty + // string then reserve the bytes for later population. + std::string body_string; + body_string.resize(read_avail); + TSIOBufferReaderCopy(post_buffer_reader, body_string.data(), read_avail); + TSIOBufferReaderFree(post_buffer_reader); + + TSDebug(debug_tag, "Returning %ld body bytes", body_string.size()); + return body_string; +} + +int +TransactionData::request_buffer_handler(TSCont contp, TSEvent event, void *edata) +{ + TSHttpTxn txnp = (TSHttpTxn)(edata); + + if (event == TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE) { + const std::string body = request_body_get(txnp); + TSDebug(debug_tag, "Got a request body of size %zu bytes", body.size()); + + auto *txnData = static_cast(TSUserArgGet(txnp, transaction_arg_index)); + txnData->request_body = body; + TSContDestroy(contp); + } else { + TSDebug(debug_tag, "Request buffer handler received an unrecognized event: %d", event); + } + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; +} + void TransactionData::initialize_default_sensitive_field() { @@ -79,17 +123,17 @@ TransactionData::get_sensitive_field_description() } bool -TransactionData::init(sensitive_fields_t &&new_fields) +TransactionData::init(bool dump_body, sensitive_fields_t &&new_fields) { sensitive_fields = std::move(new_fields); - return init_helper(); + return init_helper(dump_body); } bool -TransactionData::init() +TransactionData::init(bool dump_body) { sensitive_fields = default_sensitive_fields; - return init_helper(); + return init_helper(dump_body); } std::string_view @@ -115,6 +159,18 @@ TransactionData::write_content_node(int64_t num_body_bytes) return std::string(R"(,"content":{"encoding":"plain","size":)" + std::to_string(num_body_bytes) + '}'); } +std::string +TransactionData::write_content_node(std::string_view body) +{ + std::ostringstream content_node; + content_node << R"(,"content":{"encoding":"plain","size":)" + std::to_string(body.length()); + if (!body.empty()) { + content_node << R"(,"data":")" + escape_json(std::string(body)) + R"(")"; + } + content_node << '}'; + return content_node.str(); +} + std::string TransactionData::write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_loc) { @@ -216,8 +272,10 @@ TransactionData::remove_scheme_prefix(std::string_view url) } bool -TransactionData::init_helper() +TransactionData::init_helper(bool _dump_body) { + dump_body = _dump_body; + TSDebug(debug_tag, "Dumping body bytes: %s", dump_body ? "true" : "false"); initialize_default_sensitive_field(); const std::string sensitive_fields_string = get_sensitive_field_description(); TSDebug(debug_tag, "Sensitive fields for which generic values will be dumped: %s", sensitive_fields_string.c_str()); @@ -307,6 +365,10 @@ TransactionData::global_transaction_handler(TSCont contp, TSEvent event, void *e txnData->txn_json += R"(,"client-request":{)" + txnData->write_message_node_no_content(buffer, hdr_loc); TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); buffer = nullptr; + if (dump_body) { + TSHttpTxnConfigIntSet(txnp, TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, 1); + TSHttpTxnHookAdd(txnp, TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, TSContCreate(request_buffer_handler, TSMutexCreate())); + } } break; } @@ -331,7 +393,11 @@ TransactionData::global_transaction_handler(TSCont contp, TSEvent event, void *e TSMBuffer buffer; TSMLoc hdr_loc; if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &buffer, &hdr_loc)) { - txnData->txn_json += txnData->write_content_node(TSHttpTxnClientReqBodyBytesGet(txnp)) + "}"; + if (dump_body) { + txnData->txn_json += txnData->write_content_node(txnData->request_body) + "}"; + } else { + txnData->txn_json += txnData->write_content_node(TSHttpTxnClientReqBodyBytesGet(txnp)) + "}"; + } TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); buffer = nullptr; } diff --git a/plugins/experimental/traffic_dump/transaction_data.h b/plugins/experimental/traffic_dump/transaction_data.h index b3a13561ae7..a57af011fa2 100644 --- a/plugins/experimental/traffic_dump/transaction_data.h +++ b/plugins/experimental/traffic_dump/transaction_data.h @@ -42,7 +42,10 @@ class TransactionData /** The string for the JSON content of this transaction. */ std::string txn_json; - /** The '"protocol" node for this transaction's server-side connection. */ + /** The string of the client request body, if dump_body is true. */ + std::string request_body; + + /** The "protocol" node for this transaction's server-side conection. */ std::string server_protocol_description; // The index to be used for the TS API for storing this TransactionData on a @@ -53,17 +56,28 @@ class TransactionData /// whose values will be replaced with auto-generated generic content. static sensitive_fields_t sensitive_fields; + /// Whether the user configured the dumping of body content. + static bool dump_body; + public: /** Initialize TransactionData, using the provided sensitive fields. + * + * @param[in] dump_body Whether to dump body content. + * + * @param[in] sensitive_fields_t The HTTP fields considered to have sensitive + * data. * * @return True if initialization is successful, false otherwise. */ - static bool init(sensitive_fields_t &&sensitive_fields); + static bool init(bool dump_body, sensitive_fields_t &&sensitive_fields); /** Initialize TransactionData, using default sensitive fields. + * + * @param[in] dump_body Whether to dump body content. + * * @return True if initialization is successful, false otherwise. */ - static bool init(); + static bool init(bool dump_body); /// Read the txn information from TSMBuffer and write the header information. /// This function does not write the content node. @@ -77,8 +91,14 @@ class TransactionData static int global_transaction_handler(TSCont contp, TSEvent event, void *edata); private: - /** Common logic for the init overloads. */ - static bool init_helper(); + /** Common logic for the init overloads. + * + * @param[in] dump_body Whether the user configured the dumping of body + * content. + * + * @return True if initialization is successful, false otherwise. + */ + static bool init_helper(bool dump_body); /** Initialize the generic sensitive field to be dumped. This is used instead * of the sensitive field values seen on the wire. @@ -91,6 +111,20 @@ class TransactionData */ static std::string get_sensitive_field_description(); + /** Retrieve the request body from the transaction. + * + * @param[in] txnp The transaction from which to retrieve the request body. + * + * @return The request body string. + */ + static std::string request_body_get(TSHttpTxn txnp); + + /** The callback for gathering request body data. + * + * @note This is only called if the user enabled dump_body. + */ + static int request_buffer_handler(TSCont contp, TSEvent event, void *edata); + /** Inspect the field to see whether it is sensitive and return a generic value * of equal size to the original if it is. * @@ -108,6 +142,14 @@ class TransactionData /// "size" std::string write_content_node(int64_t num_body_bytes); + /// Write the content JSON node for an HTTP message. + // + /// "content" + /// "encoding" + /// "size" + /// "data" + std::string write_content_node(std::string_view body); + /** Remove the scheme prefix from the url. * * @return The view without the scheme prefix. diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index ae4b5a03adf..b374a335e98 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -1616,7 +1616,7 @@ plugins required to work with sni_routing. // void HttpSM::handle_api_return() // // Figures out what to do after calling api callouts -// have finished. This messy and I would like +// have finished. This is messy and I would like // to come up with a cleaner way to handle the api // return. The way we are doing things also makes a // mess of set_next_state() @@ -3619,7 +3619,14 @@ HttpSM::tunnel_handler_post_server(int event, HttpTunnelConsumer *c) { STATE_ENTER(&HttpSM::tunnel_handler_post_server, event); - server_request_body_bytes = c->bytes_written; + // If is_using_post_buffer has been used, then this handler gets called + // twice, once with the buffered request body bytes and a second time with + // the (now) zero length user agent buffer. See wait_for_full_body where + // these bytes are read. Don't clobber the server_request_body_bytes with + // zero on that second read. + if (server_request_body_bytes == 0) { + server_request_body_bytes = c->bytes_written; + } switch (event) { case VC_EVENT_EOS: @@ -5830,9 +5837,16 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) // Next order of business if copy the remaining data from the // header buffer into new buffer - client_request_body_bytes = post_buffer->write(ua_buffer_reader, chunked ? ua_buffer_reader->read_avail() : post_bytes); + int64_t body_bytes = post_buffer->write(ua_buffer_reader, chunked ? ua_buffer_reader->read_avail() : post_bytes); - ua_buffer_reader->consume(client_request_body_bytes); + // If is_using_post_buffer has been used, then client_request_body_bytes + // will have already been set in wait_for_full_body and there will be + // zero bytes in this user agent buffer. We don't want to clobber + // client_request_body_bytes with a zero value here in those cases. + if (client_request_body_bytes == 0) { + client_request_body_bytes = body_bytes; + } + ua_buffer_reader->consume(body_bytes); p = tunnel.add_producer(ua_entry->vc, post_bytes - transfered_bytes, buf_start, &HttpSM::tunnel_handler_post_ua, HT_HTTP_CLIENT, "user agent post"); } diff --git a/tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py b/tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py index 7e84e8914a4..4d96a76a01a 100644 --- a/tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py +++ b/tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py @@ -17,7 +17,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os +Test.SkipUnless( + Condition.PluginExists('request_buffer.so'), +) def int_to_hex_string(int_value): @@ -35,7 +37,7 @@ def int_to_hex_string(int_value): >>> int_to_hex_string(17) 'f1' ''' - if type(int_value) != int: + if not isinstance(int_value, int): raise ValueError("Input should be an int type.") return hex(int_value).split('x')[1] @@ -61,14 +63,14 @@ def setupOriginServer(self): "Server: microserver\r\n" "Content-Length: {}\r\n\r\n" "Connection: close\r\n\r\n".format( - len(content_length_response_body)), + len(content_length_response_body)), "timestamp": "1469733493.993", "body": content_length_response_body} self._server.addResponse("sessionlog.json", request_header, response_header) self.chunked_request_body = "chunked request" self.encoded_chunked_request = "{0}\r\n{1}\r\n0\r\n\r\n".format( - int_to_hex_string(len(self.chunked_request_body)), self.chunked_request_body) + int_to_hex_string(len(self.chunked_request_body)), self.chunked_request_body) request_header2 = {"headers": "POST /chunked HTTP/1.1\r\n" "Transfer-Encoding: chunked\r\n" "Host: www.example.com\r\n" @@ -77,7 +79,7 @@ def setupOriginServer(self): "body": self.encoded_chunked_request} self.chunked_response_body = "chunked response" self.encoded_chunked_response = "{0}\r\n{1}\r\n0\r\n\r\n".format( - int_to_hex_string(len(self.chunked_response_body)), self.chunked_response_body) + int_to_hex_string(len(self.chunked_response_body)), self.chunked_response_body) response_header2 = {"headers": "HTTP/1.1 200 OK\r\n" "Transfer-Encoding: chunked\r\n" "Server: microserver\r\n" @@ -91,7 +93,7 @@ def setupTS(self): self._ts.Disk.remap_config.AddLine( 'map / http://127.0.0.1:{0}'.format(self._server.Variables.Port) ) - Test.PreparePlugin(os.path.join(Test.Variables.AtsTestToolsDir, 'plugins', 'request_buffer.c'), self._ts) + Test.PrepareInstalledPlugin('request_buffer.so', self._ts) self._ts.Disk.records_config.update({ 'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'request_buffer', @@ -101,24 +103,21 @@ def setupTS(self): }) self._ts.Streams.stderr = Testers.ContainsExpression( - "request_buffer_plugin gets the request body with length\[{0}\]: \[{1}\]".format( - len(self.content_length_request_body), self.content_length_request_body), + r"request_buffer_plugin gets the request body with length\[{0}\]".format( + len(self.content_length_request_body)), "Verify that the plugin parsed the content-length request body data.") self._ts.Streams.stderr += Testers.ContainsExpression( - "request_buffer_plugin gets the request body with length\[{0}\]: \[".format( + r"request_buffer_plugin gets the request body with length\[{0}\]".format( len(self.encoded_chunked_request)), "Verify that the plugin parsed the chunked request body.") - self._ts.Streams.stderr += Testers.ContainsExpression( - "^{}".format(self.chunked_request_body), - "Verify that the plugin parsed the chunked request body data.") def run(self): tr = Test.AddTestRun() # Send both a Content-Length request and a chunked-encoded request. tr.Processes.Default.Command = ( - 'curl -v http://127.0.0.1:{0}/contentlength -d "{1}" ; ' - 'curl -v http://127.0.0.1:{0}/chunked -H "Transfer-Encoding: chunked" -d "{2}"'.format( - self._ts.Variables.port, self.content_length_request_body, self.chunked_request_body)) + 'curl -v http://127.0.0.1:{0}/contentlength -d "{1}" --next ' + '-v http://127.0.0.1:{0}/chunked -H "Transfer-Encoding: chunked" -d "{2}"'.format( + self._ts.Variables.port, self.content_length_request_body, self.chunked_request_body)) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.StartBefore(self._server) tr.Processes.Default.StartBefore(Test.Processes.ts) diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/200_get.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/200_get.gold new file mode 100644 index 00000000000..614c1392e4b --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/200_get.gold @@ -0,0 +1,12 @@ +`` +> GET /`` HTTP/1.1 +> Host: www.notls.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< HTTP/1.1 200 OK +< Content-Length: 0 +< Date: `` +`` +< Server: ATS/`` +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/200.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/200_get_sensitive_field.gold similarity index 100% rename from tests/gold_tests/pluginTest/traffic_dump/gold/200.gold rename to tests/gold_tests/pluginTest/traffic_dump/gold/200_get_sensitive_field.gold diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/200_post.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/200_post.gold new file mode 100644 index 00000000000..0e5bb5f6fad --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/200_post.gold @@ -0,0 +1,12 @@ +`` +> POST /`` HTTP/1.1 +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< HTTP/1.1 200 OK +< Content-Length: `` +< Date: `` +< Age: `` +< Server: ATS/`` +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py index e6e95a1bfa0..d450f7bff66 100644 --- a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py +++ b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py @@ -18,6 +18,7 @@ # limitations under the License. import os + Test.Summary = ''' Verify traffic_dump functionality. ''' @@ -159,6 +160,9 @@ ts.Streams.stderr += Testers.ContainsExpression( "Finish a session with log file of.*bytes", "Verify traffic_dump sees the end of sessions and accounts for it.") +ts.Streams.stderr += Testers.ContainsExpression( + "Dumping body bytes: false", + "Verify that dumping body bytes is enabled.") # Set up the json replay file expectations. replay_file_session_1 = os.path.join(replay_dir, "127", "0000000000000000") @@ -196,7 +200,7 @@ '-H"Host: www.notls.com" -H"X-Request-1: ultra_sensitive" --verbose'.format( ts.Variables.port)) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Streams.stderr = "gold/200.gold" +tr.Processes.Default.Streams.stderr = "gold/200_get_sensitive_field.gold" tr.StillRunningAfter = server tr.StillRunningAfter = ts http_protocols = "tcp,ip" @@ -208,7 +212,7 @@ '-H"X-Request-2: also_very_sensitive" --verbose'.format( ts.Variables.port)) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Streams.stderr = "gold/200.gold" +tr.Processes.Default.Streams.stderr = "gold/200_get_sensitive_field.gold" tr.StillRunningAfter = server tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_ip_filter.test.py b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_ip_filter.test.py new file mode 100644 index 00000000000..f5cdafdcb57 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_ip_filter.test.py @@ -0,0 +1,150 @@ +""" +Verify traffic_dump IP filter functionality. +""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Verify traffic_dump IP filter functionality. +''' + +Test.SkipUnless( + Condition.PluginExists('traffic_dump.so'), +) + +# Configure the origin server. +server = Test.MakeOriginServer("server", both=True) + +request_header = {"headers": "GET /empty HTTP/1.1\r\n" + "Host: www.notls.com\r\n" + "Content-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Content-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) + + +def get_common_ats_process(name, plugin_command, replay_exists): + """ + Get an ATS process with some common configuration. + + These tests have different log expectations, but have generally the same + ATS Process configuration. This function returns a Process with that common + configuration. + + Returns: + A configured ATS Process. + The replay file. + """ + ts = Test.MakeATSProcess(name) + replay_dir = os.path.join(ts.RunDirectory, name, "log") + ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'traffic_dump', + }) + ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) + ) + # Configure traffic_dump as specified. + ts.Disk.plugin_config.AddLine(plugin_command.format(replay_dir)) + + ts_replay_file_session_1 = os.path.join(replay_dir, "127", "0000000000000000") + ts.Disk.File(ts_replay_file_session_1, exists=replay_exists) + return ts, ts_replay_file_session_1 + + +# +# Test 1: Verify -4 works for a specified address. +# +tr = Test.AddTestRun("Verify that -4 matches 127.0.0.1 as expected") +ts1, ts1_replay_file = get_common_ats_process( + "ts1", + 'traffic_dump.so --logdir {0} --sample 1 --limit 1000000000 -4 127.0.0.1', + replay_exists=True) +ts1.Streams.stderr += Testers.ContainsExpression( + "Filtering to only dump connections with ip: 127.0.0.1", + "Verify the IP filter status message.") + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts1) +tr.Processes.Default.Command = \ + ('curl http://127.0.0.1:{0}/empty -H"Host: www.notls.com" --verbose'.format( + ts1.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_get.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts1 + +# Verify that the expected request body was recorded. +tr = Test.AddTestRun("Verify that the expected request body was recorded.") +verify_replay = "verify_replay.py" +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = "python3 {0} {1} {2}".format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + ts1_replay_file) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts1 + +# +# Test 2: Verify -4 filters out other addresses. +# +tr = Test.AddTestRun("Verify that -4 filters out our non-matching IP as expected") +ts2, ts2_replay_file = get_common_ats_process( + "ts2", + 'traffic_dump.so --logdir {0} --sample 1 --limit 1000000000 -4 1.2.3.4', + replay_exists=False) +ts2.Streams.stderr += Testers.ContainsExpression( + "Filtering to only dump connections with ip: 1.2.3.4", + "Verify the IP filter status message.") + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts2) +tr.Processes.Default.Command = \ + ('curl http://127.0.0.1:{0}/empty -H"Host: www.notls.com" --verbose'.format( + ts2.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_get.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts2 + +# +# Test 3: Verify -4 recognizes an invalid IP address string. +# +tr = Test.AddTestRun("Verify that -4 detects an invalid IP string") +invalid_ip = "this_is_not_a_valid_ip_string" +ts3, ts3_replay_file = get_common_ats_process( + "ts3", + 'traffic_dump.so --logdir {0} --sample 1 --limit 1000000000 -4 ' + invalid_ip, + replay_exists=False) +ts3.Disk.diags_log.Content = Testers.ContainsExpression( + "Problems parsing IP filter address argument: {}".format(invalid_ip), + "Verify traffic_dump detects an invalid IPv4 address.") + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts3) +tr.Processes.Default.Command = \ + ('curl http://127.0.0.1:{0}/empty -H"Host: www.notls.com" --verbose'.format( + ts3.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_get.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts3 diff --git a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_request_body.test.py b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_request_body.test.py new file mode 100644 index 00000000000..bcbb9e7781b --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_request_body.test.py @@ -0,0 +1,182 @@ +""" +Verify traffic_dump request body functionality. +""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Verify traffic_dump request body functionality. +''' + +Test.SkipUnless( + Condition.PluginExists('traffic_dump.so'), +) + +# Configure the origin server. +server = Test.MakeOriginServer("server", both=True) + +request_header = {"headers": "GET /empty HTTP/1.1\r\n" + "Host: www.notls.com\r\n" + "Content-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Content-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) +request_header = {"headers": "POST /with_content_length HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Content-Length: 4\r\n\r\n", + "timestamp": "1469733493.993", "body": "1234"} +response_header = {"headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Content-Length: 4\r\n\r\n", + "timestamp": "1469733493.993", "body": "1234"} +server.addResponse("sessionfile.log", request_header, response_header) + + +def get_common_ats_process(name): + """ + Get an ATS process with some common configuration. + + These tests have different log expectations, but have generally the same + ATS Process configuration. This function returns a Process with that common + configuration. + + Returns: + A configured ATS Process. + The replay file. + """ + ts = Test.MakeATSProcess(name) + replay_dir = os.path.join(ts.RunDirectory, name, "log") + ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'traffic_dump', + }) + ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) + ) + # Configure traffic_dump to dump body bytes (-b). + ts.Disk.plugin_config.AddLine( + 'traffic_dump.so --logdir {0} --sample 1 --limit 1000000000 -b'.format(replay_dir) + ) + + ts_replay_file_session_1 = os.path.join(replay_dir, "127", "0000000000000000") + ts.Disk.File(ts_replay_file_session_1, exists=True) + + ts.Streams.stderr = Testers.ContainsExpression( + "Dumping body bytes: true", + "Verify that dumping body bytes is enabled.") + return ts, ts_replay_file_session_1 + + +# +# Test 1: Verify a request without a body is dumped correctly. +# +tr = Test.AddTestRun("An empty request body is handled correctly.") +ts1, ts1_replay_file = get_common_ats_process("ts1") + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts1) +tr.Processes.Default.Command = 'curl http://127.0.0.1:{0}/empty -H"Host: www.notls.com" --verbose'.format( + ts1.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_get.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts1 + +tr = Test.AddTestRun("Verify the json content of the first session") +verify_replay = "verify_replay.py" +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = 'python3 {0} {1} {2}'.format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + ts1_replay_file) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts1 + +# +# Test 2: Verify request body can be dumped. +# +tr = Test.AddTestRun("Verify body bytes can be dumped") +ts2, ts2_replay_file = get_common_ats_process("ts2") +ts2.Streams.stderr += Testers.ContainsExpression( + "Got a request body of size 4 bytes", + "Verify logging of the dumped body bytes.") + +request_body = "1234" + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts2) +tr.Processes.Default.Command = \ + ('curl http://127.0.0.1:{0}/with_content_length -H"Host: www.example.com" ' + '--verbose -d "{1}"'.format( + ts2.Variables.port, + request_body)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_post.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts2 + +# Verify that the expected request body was recorded. +tr = Test.AddTestRun("Verify that the expected request body was recorded.") +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = "python3 {0} {1} {2} --request_body {3}".format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + ts2_replay_file, + request_body) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts2 + +# +# Test 3: Verify request body bytes are escaped. +# +tr = Test.AddTestRun("Verify body bytes are dumped") +ts3, ts3_replay_file = get_common_ats_process("ts3") +ts3.Streams.stderr += Testers.ContainsExpression( + "Got a request body of size 5 bytes", + "Verify logging of the dumped body bytes.") + +request_body = '12"34' + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts3) +tr.Processes.Default.Command = \ + ("curl http://127.0.0.1:{0}/with_content_length -H'Host: www.example.com' " + "--verbose -d '{1}'".format( + ts3.Variables.port, + request_body)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_post.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts3 + +# Verify that the expected request body was recorded. +tr = Test.AddTestRun("Verify that the expected request body was recorded.") +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = "python3 {0} {1} {2} --request_body '{3}'".format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + ts3_replay_file, + r'12"34') +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts3 diff --git a/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py b/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py index a741e810f37..74851188942 100644 --- a/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py +++ b/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py @@ -254,11 +254,42 @@ def verify_server_tls_features(replay_json, expected_tls_features): return True +def verify_client_request_body_bytes(replay_json, expected_body_bytes): + """ + Verify that the replay file has the specified body bytes in it. + """ + try: + received_body_bytes = replay_json['sessions'][0]['transactions'][0]['client-request']['content']['data'] + except KeyError: + print("The replay file did not have a body element in the first transaction.") + return False + + if received_body_bytes != expected_body_bytes: + print("Expected body bytes of '{0}' but got '{1}'".format(expected_body_bytes, received_body_bytes)) + return False + + # If the client request had that many bytes, verify that the proxy-request + # does too. This is not guaranteed to always be true, but for our autest it + # currently holds and this check is worthwhile. + try: + proxy_request_body_size = replay_json['sessions'][0]['transactions'][0]['proxy-request']['content']['size'] + except KeyError: + print("The replay file did not have a proxgy-rerquest content size element in the first transaction.") + return False + + if int(proxy_request_body_size) != len(expected_body_bytes): + print("Expected the proxy-request content size to be '{0}' but got '{1}'".format( + len(expected_body_bytes), proxy_request_body_size)) + return False + + return True + + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("schema_file", type=argparse.FileType('r'), - help="The schema in which to interpret validate the replay file.") + help="The schema in which to validate the replay file.") parser.add_argument("replay_file", type=argparse.FileType('r'), help="The replay file to validate.") @@ -278,6 +309,9 @@ def parse_args(): help="The TLS values to expect for the client connection.") parser.add_argument("--server-tls-features", help="The TLS values to expect for the server connection.") + parser.add_argument("--request_body", + type=str, + help="Verify that the client request has the specified body bytes.") return parser.parse_args() @@ -323,6 +357,9 @@ def main(): if args.server_tls_features and not verify_server_tls_features(replay_json, args.server_tls_features): return 1 + if args.request_body and not verify_client_request_body_bytes(replay_json, args.request_body): + return 1 + return 0