diff --git a/example/plugins/c-api/request_buffer/request_buffer.c b/example/plugins/c-api/request_buffer/request_buffer.c index 69ab0c6269d..2214cda5243 100644 --- a/example/plugins/c-api/request_buffer/request_buffer.c +++ b/example/plugins/c-api/request_buffer/request_buffer.c @@ -67,7 +67,7 @@ request_buffer_plugin(TSCont contp, TSEvent event, void *edata) { TSDebug(PLUGIN_NAME, "request_buffer_plugin starting, event[%d]", event); TSHttpTxn txnp = (TSHttpTxn)(edata); - if (event == TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE) { + if (event == TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE) { int len = 0; char *body = request_body_get(txnp, &len); TSDebug(PLUGIN_NAME, "request_buffer_plugin gets the request body with length[%d]", len); diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index b2bf28140ce..017fd54179d 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -210,6 +210,122 @@ typedef enum { TS_HTTP_STATUS_NETWORK_AUTHENTICATION_REQUIRED = 511 } TSHttpStatus; +/** + TSEvents are sent to continuations when they are called back. + The TSEvent provides the continuation's handler function with + information about the callback. Based on the event it receives, + the handler function can decide what to do. + + */ +typedef enum { + TS_EVENT_NONE = 0, + TS_EVENT_IMMEDIATE = 1, + TS_EVENT_TIMEOUT = 2, + TS_EVENT_ERROR = 3, + TS_EVENT_CONTINUE = 4, + + TS_EVENT_VCONN_READ_READY = 100, + TS_EVENT_VCONN_WRITE_READY = 101, + TS_EVENT_VCONN_READ_COMPLETE = 102, + TS_EVENT_VCONN_WRITE_COMPLETE = 103, + TS_EVENT_VCONN_EOS = 104, + TS_EVENT_VCONN_INACTIVITY_TIMEOUT = 105, + TS_EVENT_VCONN_ACTIVE_TIMEOUT = 106, + TS_EVENT_VCONN_START = 107, + TS_EVENT_VCONN_CLOSE = 108, + TS_EVENT_VCONN_OUTBOUND_START = 109, + TS_EVENT_VCONN_OUTBOUND_CLOSE = 110, + TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible + + TS_EVENT_NET_CONNECT = 200, + TS_EVENT_NET_CONNECT_FAILED = 201, + TS_EVENT_NET_ACCEPT = 202, + TS_EVENT_NET_ACCEPT_FAILED = 204, + + TS_EVENT_INTERNAL_206 = 206, + TS_EVENT_INTERNAL_207 = 207, + TS_EVENT_INTERNAL_208 = 208, + TS_EVENT_INTERNAL_209 = 209, + TS_EVENT_INTERNAL_210 = 210, + TS_EVENT_INTERNAL_211 = 211, + TS_EVENT_INTERNAL_212 = 212, + + TS_EVENT_HOST_LOOKUP = 500, + + TS_EVENT_CACHE_OPEN_READ = 1102, + TS_EVENT_CACHE_OPEN_READ_FAILED = 1103, + TS_EVENT_CACHE_OPEN_WRITE = 1108, + TS_EVENT_CACHE_OPEN_WRITE_FAILED = 1109, + TS_EVENT_CACHE_REMOVE = 1112, + TS_EVENT_CACHE_REMOVE_FAILED = 1113, + TS_EVENT_CACHE_SCAN = 1120, + TS_EVENT_CACHE_SCAN_FAILED = 1121, + TS_EVENT_CACHE_SCAN_OBJECT = 1122, + TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED = 1123, + TS_EVENT_CACHE_SCAN_OPERATION_FAILED = 1124, + TS_EVENT_CACHE_SCAN_DONE = 1125, + TS_EVENT_CACHE_LOOKUP = 1126, + TS_EVENT_CACHE_READ = 1127, + TS_EVENT_CACHE_DELETE = 1128, + TS_EVENT_CACHE_WRITE = 1129, + TS_EVENT_CACHE_WRITE_HEADER = 1130, + TS_EVENT_CACHE_CLOSE = 1131, + TS_EVENT_CACHE_LOOKUP_READY = 1132, + TS_EVENT_CACHE_LOOKUP_COMPLETE = 1133, + TS_EVENT_CACHE_READ_READY = 1134, + TS_EVENT_CACHE_READ_COMPLETE = 1135, + + TS_EVENT_INTERNAL_1200 = 1200, + + TS_EVENT_SSL_SESSION_GET = 2000, + TS_EVENT_SSL_SESSION_NEW = 2001, + TS_EVENT_SSL_SESSION_REMOVE = 2002, + + TS_EVENT_AIO_DONE = 3900, + + TS_EVENT_HTTP_CONTINUE = 60000, + TS_EVENT_HTTP_ERROR = 60001, + TS_EVENT_HTTP_READ_REQUEST_HDR = 60002, + TS_EVENT_HTTP_OS_DNS = 60003, + TS_EVENT_HTTP_SEND_REQUEST_HDR = 60004, + TS_EVENT_HTTP_READ_CACHE_HDR = 60005, + TS_EVENT_HTTP_READ_RESPONSE_HDR = 60006, + TS_EVENT_HTTP_SEND_RESPONSE_HDR = 60007, + TS_EVENT_HTTP_REQUEST_TRANSFORM = 60008, + TS_EVENT_HTTP_RESPONSE_TRANSFORM = 60009, + TS_EVENT_HTTP_SELECT_ALT = 60010, + TS_EVENT_HTTP_TXN_START = 60011, + TS_EVENT_HTTP_TXN_CLOSE = 60012, + TS_EVENT_HTTP_SSN_START = 60013, + TS_EVENT_HTTP_SSN_CLOSE = 60014, + TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, + TS_EVENT_HTTP_PRE_REMAP = 60016, + TS_EVENT_HTTP_POST_REMAP = 60017, + TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE = 60018, + TS_EVENT_HTTP_RESPONSE_CLIENT = 60019, + + TS_EVENT_LIFECYCLE_PORTS_INITIALIZED = 60100, + TS_EVENT_LIFECYCLE_PORTS_READY = 60101, + TS_EVENT_LIFECYCLE_CACHE_READY = 60102, + TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED = 60103, + TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60104, + TS_EVENT_LIFECYCLE_MSG = 60105, + TS_EVENT_LIFECYCLE_TASK_THREADS_READY = 60106, + TS_EVENT_LIFECYCLE_SHUTDOWN = 60107, + + TS_EVENT_INTERNAL_60200 = 60200, + TS_EVENT_INTERNAL_60201 = 60201, + TS_EVENT_INTERNAL_60202 = 60202, + TS_EVENT_SSL_CERT = 60203, + TS_EVENT_SSL_SERVERNAME = 60204, + TS_EVENT_SSL_VERIFY_SERVER = 60205, + TS_EVENT_SSL_VERIFY_CLIENT = 60206, + TS_EVENT_SSL_CLIENT_HELLO = 60207, + + TS_EVENT_MGMT_UPDATE = 60300 +} TSEvent; +#define TS_EVENT_HTTP_READ_REQUEST_PRE_REMAP TS_EVENT_HTTP_PRE_REMAP /* backwards compat */ + /** This set of enums represents the possible hooks where you can set up continuation callbacks. The functions used to register a @@ -262,25 +378,48 @@ typedef enum { TS_HTTP_LAST_HOOK _must_ be the last element. Only right place to insert a new element is just before TS_HTTP_LAST_HOOK. + @note The TS_HTTP hooks below have to be in the same order as their + corresponding TS_EVENT counterparts. We use this in calculating the + corresponding event from a hook ID by summing + TS_EVENT_HTTP_READ_REQUEST_HDR with the hook ID (see the invoke call in + HttpSM::state_api_callout). For example, the following expression must + be true: + + TS_EVENT_HTTP_TXN_CLOSE == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_TXN_CLOSE_HOOK + */ typedef enum { - TS_HTTP_READ_REQUEST_HDR_HOOK, - TS_HTTP_OS_DNS_HOOK, - TS_HTTP_SEND_REQUEST_HDR_HOOK, - TS_HTTP_READ_CACHE_HDR_HOOK, - TS_HTTP_READ_RESPONSE_HDR_HOOK, - TS_HTTP_SEND_RESPONSE_HDR_HOOK, - TS_HTTP_REQUEST_TRANSFORM_HOOK, - TS_HTTP_RESPONSE_TRANSFORM_HOOK, - TS_HTTP_SELECT_ALT_HOOK, - TS_HTTP_TXN_START_HOOK, - TS_HTTP_TXN_CLOSE_HOOK, - TS_HTTP_SSN_START_HOOK, - TS_HTTP_SSN_CLOSE_HOOK, - TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, - TS_HTTP_PRE_REMAP_HOOK, - TS_HTTP_POST_REMAP_HOOK, - TS_HTTP_RESPONSE_CLIENT_HOOK, +/* + The following macro helps maintain the relationship between the TS_HTTP + hooks and their TS_EVENT_HTTP counterparts as described in the above + doxygen comment. +*/ +#define REBASE(COMMON) TS_HTTP_##COMMON##_HOOK = TS_EVENT_HTTP_##COMMON - TS_EVENT_HTTP_READ_REQUEST_HDR + REBASE(READ_REQUEST_HDR), // TS_HTTP_READ_REQUEST_HDR_HOOK + REBASE(OS_DNS), // TS_HTTP_OS_DNS_HOOK + REBASE(SEND_REQUEST_HDR), // TS_HTTP_SEND_REQUEST_HDR_HOOK + REBASE(READ_CACHE_HDR), // TS_HTTP_READ_CACHE_HDR_HOOK + REBASE(READ_RESPONSE_HDR), // TS_HTTP_READ_RESPONSE_HDR_HOOK + REBASE(SEND_RESPONSE_HDR), // TS_HTTP_SEND_RESPONSE_HDR_HOOK + REBASE(REQUEST_TRANSFORM), // TS_HTTP_REQUEST_TRANSFORM_HOOK + REBASE(RESPONSE_TRANSFORM), // TS_HTTP_RESPONSE_TRANSFORM_HOOK + REBASE(SELECT_ALT), // TS_HTTP_SELECT_ALT_HOOK + REBASE(TXN_START), // TS_HTTP_TXN_START_HOOK + REBASE(TXN_CLOSE), // TS_HTTP_TXN_CLOSE_HOOK + REBASE(SSN_START), // TS_HTTP_SSN_START_HOOK + REBASE(SSN_CLOSE), // TS_HTTP_SSN_CLOSE_HOOK + REBASE(CACHE_LOOKUP_COMPLETE), // TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK + REBASE(PRE_REMAP), // TS_HTTP_PRE_REMAP_HOOK + REBASE(POST_REMAP), // TS_HTTP_POST_REMAP_HOOK + REBASE(REQUEST_BUFFER_READ_COMPLETE), // TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK + REBASE(RESPONSE_CLIENT), // TS_HTTP_RESPONSE_CLIENT_HOOK +#undef REBASE + + // NOTE: + // If adding any TS_HTTP hooks, be sure to understand the note above in the + // doxygen comment about the ordering of these enum values with respect to + // their corresponding TS_EVENT values. + // Putting the SSL hooks in the same enum space // So both sets of hooks can be set by the same Hook function TS_SSL_FIRST_HOOK, @@ -298,7 +437,6 @@ typedef enum { TS_VCONN_OUTBOUND_START_HOOK, TS_VCONN_OUTBOUND_CLOSE_HOOK, TS_SSL_LAST_HOOK = TS_VCONN_OUTBOUND_CLOSE_HOOK, - TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, TS_HTTP_LAST_HOOK } TSHttpHookID; @@ -387,121 +525,6 @@ typedef enum { TS_LIFECYCLE_LAST_HOOK } TSLifecycleHookID; -/** - TSEvents are sent to continuations when they are called back. - The TSEvent provides the continuation's handler function with - information about the callback. Based on the event it receives, - the handler function can decide what to do. - - */ -typedef enum { - TS_EVENT_NONE = 0, - TS_EVENT_IMMEDIATE = 1, - TS_EVENT_TIMEOUT = 2, - TS_EVENT_ERROR = 3, - TS_EVENT_CONTINUE = 4, - - TS_EVENT_VCONN_READ_READY = 100, - TS_EVENT_VCONN_WRITE_READY = 101, - TS_EVENT_VCONN_READ_COMPLETE = 102, - TS_EVENT_VCONN_WRITE_COMPLETE = 103, - TS_EVENT_VCONN_EOS = 104, - TS_EVENT_VCONN_INACTIVITY_TIMEOUT = 105, - TS_EVENT_VCONN_ACTIVE_TIMEOUT = 106, - TS_EVENT_VCONN_START = 107, - TS_EVENT_VCONN_CLOSE = 108, - TS_EVENT_VCONN_OUTBOUND_START = 109, - TS_EVENT_VCONN_OUTBOUND_CLOSE = 110, - TS_EVENT_VCONN_PRE_ACCEPT = TS_EVENT_VCONN_START, // Deprecated but still compatible - - TS_EVENT_NET_CONNECT = 200, - TS_EVENT_NET_CONNECT_FAILED = 201, - TS_EVENT_NET_ACCEPT = 202, - TS_EVENT_NET_ACCEPT_FAILED = 204, - - TS_EVENT_INTERNAL_206 = 206, - TS_EVENT_INTERNAL_207 = 207, - TS_EVENT_INTERNAL_208 = 208, - TS_EVENT_INTERNAL_209 = 209, - TS_EVENT_INTERNAL_210 = 210, - TS_EVENT_INTERNAL_211 = 211, - TS_EVENT_INTERNAL_212 = 212, - - TS_EVENT_HOST_LOOKUP = 500, - - TS_EVENT_CACHE_OPEN_READ = 1102, - TS_EVENT_CACHE_OPEN_READ_FAILED = 1103, - TS_EVENT_CACHE_OPEN_WRITE = 1108, - TS_EVENT_CACHE_OPEN_WRITE_FAILED = 1109, - TS_EVENT_CACHE_REMOVE = 1112, - TS_EVENT_CACHE_REMOVE_FAILED = 1113, - TS_EVENT_CACHE_SCAN = 1120, - TS_EVENT_CACHE_SCAN_FAILED = 1121, - TS_EVENT_CACHE_SCAN_OBJECT = 1122, - TS_EVENT_CACHE_SCAN_OPERATION_BLOCKED = 1123, - TS_EVENT_CACHE_SCAN_OPERATION_FAILED = 1124, - TS_EVENT_CACHE_SCAN_DONE = 1125, - TS_EVENT_CACHE_LOOKUP = 1126, - TS_EVENT_CACHE_READ = 1127, - TS_EVENT_CACHE_DELETE = 1128, - TS_EVENT_CACHE_WRITE = 1129, - TS_EVENT_CACHE_WRITE_HEADER = 1130, - TS_EVENT_CACHE_CLOSE = 1131, - TS_EVENT_CACHE_LOOKUP_READY = 1132, - TS_EVENT_CACHE_LOOKUP_COMPLETE = 1133, - TS_EVENT_CACHE_READ_READY = 1134, - TS_EVENT_CACHE_READ_COMPLETE = 1135, - - TS_EVENT_INTERNAL_1200 = 1200, - - TS_EVENT_SSL_SESSION_GET = 2000, - TS_EVENT_SSL_SESSION_NEW = 2001, - TS_EVENT_SSL_SESSION_REMOVE = 2002, - - TS_EVENT_AIO_DONE = 3900, - - TS_EVENT_HTTP_CONTINUE = 60000, - TS_EVENT_HTTP_ERROR = 60001, - TS_EVENT_HTTP_READ_REQUEST_HDR = 60002, - TS_EVENT_HTTP_OS_DNS = 60003, - TS_EVENT_HTTP_SEND_REQUEST_HDR = 60004, - TS_EVENT_HTTP_READ_CACHE_HDR = 60005, - TS_EVENT_HTTP_READ_RESPONSE_HDR = 60006, - TS_EVENT_HTTP_SEND_RESPONSE_HDR = 60007, - TS_EVENT_HTTP_REQUEST_TRANSFORM = 60008, - TS_EVENT_HTTP_RESPONSE_TRANSFORM = 60009, - TS_EVENT_HTTP_SELECT_ALT = 60010, - TS_EVENT_HTTP_TXN_START = 60011, - TS_EVENT_HTTP_TXN_CLOSE = 60012, - TS_EVENT_HTTP_SSN_START = 60013, - TS_EVENT_HTTP_SSN_CLOSE = 60014, - TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE = 60015, - TS_EVENT_HTTP_PRE_REMAP = 60016, - TS_EVENT_HTTP_POST_REMAP = 60017, - TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE = 60018, - - TS_EVENT_LIFECYCLE_PORTS_INITIALIZED = 60100, - TS_EVENT_LIFECYCLE_PORTS_READY = 60101, - TS_EVENT_LIFECYCLE_CACHE_READY = 60102, - TS_EVENT_LIFECYCLE_SERVER_SSL_CTX_INITIALIZED = 60103, - TS_EVENT_LIFECYCLE_CLIENT_SSL_CTX_INITIALIZED = 60104, - TS_EVENT_LIFECYCLE_MSG = 60105, - TS_EVENT_LIFECYCLE_TASK_THREADS_READY = 60106, - TS_EVENT_LIFECYCLE_SHUTDOWN = 60107, - - TS_EVENT_INTERNAL_60200 = 60200, - TS_EVENT_INTERNAL_60201 = 60201, - TS_EVENT_INTERNAL_60202 = 60202, - TS_EVENT_SSL_CERT = 60203, - TS_EVENT_SSL_SERVERNAME = 60204, - TS_EVENT_SSL_VERIFY_SERVER = 60205, - TS_EVENT_SSL_VERIFY_CLIENT = 60206, - TS_EVENT_SSL_CLIENT_HELLO = 60207, - - TS_EVENT_MGMT_UPDATE = 60300 -} TSEvent; -#define TS_EVENT_HTTP_READ_REQUEST_PRE_REMAP TS_EVENT_HTTP_PRE_REMAP /* backwards compat */ - typedef enum { TS_SRVSTATE_STATE_UNDEFINED = 0, TS_SRVSTATE_ACTIVE_TIMEOUT, diff --git a/include/tscore/ink_inet.h b/include/tscore/ink_inet.h index aa6ddf057dc..63310daa5ff 100644 --- a/include/tscore/ink_inet.h +++ b/include/tscore/ink_inet.h @@ -1179,6 +1179,7 @@ struct IpAddr { /// Construct from @c sockaddr. explicit IpAddr(sockaddr const *addr) { this->assign(addr); } + explicit IpAddr(sockaddr const &addr) { this->assign(&addr); } /// Construct from @c sockaddr_in6. explicit IpAddr(sockaddr_in6 const &addr) { this->assign(ats_ip_sa_cast(&addr)); } /// Construct from @c sockaddr_in6. diff --git a/plugins/experimental/traffic_dump/json_utils.cc b/plugins/experimental/traffic_dump/json_utils.cc index 3b5fbbdc690..e45f9c753f4 100644 --- a/plugins/experimental/traffic_dump/json_utils.cc +++ b/plugins/experimental/traffic_dump/json_utils.cc @@ -132,31 +132,6 @@ esc_json_out(const char *buf, int64_t len, std::ostream &jsonfile) return len; } -/** Escape characters in a string as needed and return the resultant escaped string. - * - * @param[in] s The characters that need to be escaped. - */ -std::string -escape_json(std::string_view s) -{ - std::ostringstream o; - esc_json_out(s.data(), s.length(), o); - return o.str(); -} - -/** An escape_json overload for a char buffer. - * - * @param[in] buf The char buffer pointer with characters that need to be escaped. - * - * @param[in] size The size of the buf char array. - */ -std::string -escape_json(char const *buf, int64_t size) -{ - std::ostringstream o; - esc_json_out(buf, size, o); - return o.str(); -} } // anonymous namespace @@ -180,4 +155,20 @@ json_entry_array(std::string_view name, std::string_view value) return "[\"" + escape_json(name) + "\",\"" + escape_json(value) + "\"]"; } +std::string +escape_json(std::string_view s) +{ + std::ostringstream o; + esc_json_out(s.data(), s.length(), o); + return o.str(); +} + +std::string +escape_json(char const *buf, int64_t size) +{ + std::ostringstream o; + esc_json_out(buf, size, o); + return o.str(); +} + } // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/json_utils.h b/plugins/experimental/traffic_dump/json_utils.h index e91829cb45a..839ea081de7 100644 --- a/plugins/experimental/traffic_dump/json_utils.h +++ b/plugins/experimental/traffic_dump/json_utils.h @@ -60,4 +60,18 @@ std::string json_entry(std::string_view name, char const *value, int64_t size); */ std::string json_entry_array(std::string_view name, std::string_view value); +/** Escape characters in a string as needed and return the resultant escaped string. + * + * @param[in] s The characters that need to be escaped. + */ +std::string escape_json(std::string_view s); + +/** An escape_json overload for a char buffer. + * + * @param[in] buf The char buffer pointer with characters that need to be escaped. + * + * @param[in] size The size of the buf char array. + */ +std::string escape_json(char const *buf, int64_t size); + } // namespace traffic_dump diff --git a/plugins/experimental/traffic_dump/session_data.cc b/plugins/experimental/traffic_dump/session_data.cc index de41e09dc6a..5572a4ba009 100644 --- a/plugins/experimental/traffic_dump/session_data.cc +++ b/plugins/experimental/traffic_dump/session_data.cc @@ -196,6 +196,7 @@ std::atomic SessionData::disk_usage = 0; ts::file::path SessionData::log_directory{default_log_directory}; uint64_t SessionData::session_counter = 0; std::string SessionData::sni_filter; +std::optional SessionData::client_ip_filter = std::nullopt; int SessionData::get_session_arg_index() @@ -222,12 +223,25 @@ SessionData::set_max_disk_usage(int64_t new_max_disk_usage) } bool -SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size) +SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view ip_filter) { SessionData::log_directory = log_directory; SessionData::max_disk_usage = max_disk_usage; SessionData::sample_pool_size = sample_size; + if (!ip_filter.empty()) { + client_ip_filter.emplace(); + if (client_ip_filter->load(ip_filter)) { + TSDebug(debug_tag, "Problems parsing IP filter address argument: %.*s", static_cast(ip_filter.size()), ip_filter.data()); + TSError("[%s] Problems parsing IP filter address argument: %.*s", debug_tag, static_cast(ip_filter.size()), + ip_filter.data()); + client_ip_filter = std::nullopt; + return false; + } else { + TSDebug(debug_tag, "Filtering to only dump connections with ip: %.*s", static_cast(ip_filter.size()), ip_filter.data()); + } + } + if (TS_SUCCESS != TSUserArgIndexReserve(TS_USER_ARGS_SSN, debug_tag, "Track log related data", &session_arg_index)) { TSError("[%s] Unable to initialize plugin (disabled). Failed to reserve ssn arg.", traffic_dump::debug_tag); return false; @@ -244,9 +258,10 @@ SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_ } bool -SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view sni_filter) +SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view ip_filter, + std::string_view sni_filter) { - if (!init(log_directory, max_disk_usage, sample_size)) { + if (!init(log_directory, max_disk_usage, sample_size, ip_filter)) { return false; } SessionData::sni_filter = sni_filter; @@ -342,6 +357,25 @@ SessionData::write_transaction_to_disk(std::string_view content) return result; } +bool +SessionData::is_filtered_out(const sockaddr *session_client_ip) +{ + if (!client_ip_filter) { + // The user did not configure an IP by which to filter. + return false; + } + if (session_client_ip == nullptr) { + TSDebug(debug_tag, "Found no client IP address for session. Abort."); + return true; + } + if (session_client_ip->sa_family != AF_INET && session_client_ip->sa_family != AF_INET6) { + TSDebug(debug_tag, "IP family is not v4 nor v6. Abort."); + return true; + } + + IpAddr session_address(*session_client_ip); + return session_address != *client_ip_filter; +} int SessionData::session_aio_handler(TSCont contp, TSEvent event, void *edata) { @@ -418,12 +452,17 @@ SessionData::global_session_handler(TSCont contp, TSEvent event, void *edata) } const auto this_session_count = session_counter++; if (this_session_count % sample_pool_size != 0) { - TSDebug(debug_tag, "global_session_handler(): Ignore session %" PRId64 "...", id); + TSDebug(debug_tag, "Ignore session %" PRId64 " per the random sampling mechanism", id); break; } else if (disk_usage >= max_disk_usage) { - TSDebug(debug_tag, "global_session_handler(): Ignore session %" PRId64 "due to disk usage %" PRId64 "bytes", id, - disk_usage.load()); + TSDebug(debug_tag, "Ignore session %" PRId64 "due to disk usage %" PRId64 "bytes", id, disk_usage.load()); break; + } else { + const sockaddr *client_ip = TSHttpSsnClientAddrGet(ssnp); + if (SessionData::is_filtered_out(client_ip)) { + TSDebug(debug_tag, "Ignore session %" PRId64 " per the client's IP filter", id); + break; + } } // Beginning of a new session /// Get epoch time diff --git a/plugins/experimental/traffic_dump/session_data.h b/plugins/experimental/traffic_dump/session_data.h index b25a42f57bb..0cac9b0e751 100644 --- a/plugins/experimental/traffic_dump/session_data.h +++ b/plugins/experimental/traffic_dump/session_data.h @@ -29,6 +29,7 @@ #include #include "ts/ts.h" +#include "tscore/ink_inet.h" #include "tscore/ts_file.h" namespace traffic_dump @@ -100,6 +101,9 @@ class SessionData /// The running counter of all sessions dumped by traffic_dump. static uint64_t session_counter; + /// Only addresses with this IP will be dumped (if set). + static std::optional client_ip_filter; + public: SessionData(); ~SessionData(); @@ -111,8 +115,9 @@ class SessionData * * @return True if initialization is successful, false otherwise. */ - static bool init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size); - static bool init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view sni_filter); + static bool init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view ip_filter); + static bool init(std::string_view log_directory, int64_t max_disk_usage, int64_t sample_size, std::string_view ip_filter, + std::string_view sni_filter); /** Set the sample_pool_size to a new value. * @@ -173,6 +178,22 @@ class SessionData */ int write_to_disk_no_lock(std::string_view content); + /** Determine whether the user configured IP filter indicates this transaction + * should not be dumped. + * + * @note This also does some validity verification on the IP, making sure it is + * v4 or v6, and returns true (i.e., it should be filtered out) if it does not + * match these checks. These checks are required in order to perform the IP + * filtering logic. This function will only return true if the client has + * enabled client filtering. + * + * @param[in] session_client_ip The session's client address. + * + * @return True if the provided address should not be dumped per the user's + * configuration, false otherwise. + */ + static bool is_filtered_out(const sockaddr *session_client_ip); + /** Get the JSON string that describes the client session stack. * * @param[in] ssnp The reference to the client session. diff --git a/plugins/experimental/traffic_dump/traffic_dump.cc b/plugins/experimental/traffic_dump/traffic_dump.cc index 38ae1f17ba5..115ebeeca69 100644 --- a/plugins/experimental/traffic_dump/traffic_dump.cc +++ b/plugins/experimental/traffic_dump/traffic_dump.cc @@ -81,22 +81,33 @@ TSPluginInit(int argc, char const *argv[]) return; } + bool dump_body = false; bool sensitive_fields_were_specified = false; traffic_dump::sensitive_fields_t user_specified_fields; ts::file::path log_dir{traffic_dump::SessionData::default_log_directory}; int64_t sample_pool_size = traffic_dump::SessionData::default_sample_pool_size; int64_t max_disk_usage = traffic_dump::SessionData::default_max_disk_usage; std::string sni_filter; + std::string client_ip_filter; /// Commandline options - static const struct option longopts[] = { - {"logdir", required_argument, nullptr, 'l'}, {"sample", required_argument, nullptr, 's'}, - {"limit", required_argument, nullptr, 'm'}, {"sensitive-fields", required_argument, nullptr, 'f'}, - {"sni-filter", required_argument, nullptr, 'n'}, {nullptr, no_argument, nullptr, 0}}; - int opt = 0; + static const struct option longopts[] = {{"dump_body", no_argument, nullptr, 'b'}, + {"logdir", required_argument, nullptr, 'l'}, + {"sample", required_argument, nullptr, 's'}, + {"limit", required_argument, nullptr, 'm'}, + {"sensitive-fields", required_argument, nullptr, 'f'}, + {"sni-filter", required_argument, nullptr, 'n'}, + {"client_ipv4", required_argument, nullptr, '4'}, + {"client_ipv6", required_argument, nullptr, '6'}, + {nullptr, no_argument, nullptr, 0}}; + int opt = 0; while (opt >= 0) { - opt = getopt_long(argc, const_cast(argv), "l:", longopts, nullptr); + opt = getopt_long(argc, const_cast(argv), "bf:l:s:m:n:4:6", longopts, nullptr); switch (opt) { + case 'b': { + dump_body = true; + break; + } case 'f': { // --sensitive-fields takes a comma-separated list of HTTP fields that // are sensitive. The field values for these fields will be replaced @@ -133,6 +144,12 @@ TSPluginInit(int argc, char const *argv[]) } case 'm': { max_disk_usage = static_cast(std::strtol(optarg, nullptr, 0)); + break; + } + case '4': + case '6': { + client_ip_filter = std::string(optarg); + break; } case -1: case '?': @@ -148,26 +165,26 @@ TSPluginInit(int argc, char const *argv[]) log_dir = ts::file::path(TSInstallDirGet()) / log_dir; } if (sni_filter.empty()) { - if (!traffic_dump::SessionData::init(log_dir.view(), max_disk_usage, sample_pool_size)) { + if (!traffic_dump::SessionData::init(log_dir.view(), max_disk_usage, sample_pool_size, client_ip_filter)) { TSError("[%s] Failed to initialize session state.", traffic_dump::debug_tag); return; } } else { - if (!traffic_dump::SessionData::init(log_dir.view(), max_disk_usage, sample_pool_size, sni_filter)) { + if (!traffic_dump::SessionData::init(log_dir.view(), max_disk_usage, sample_pool_size, client_ip_filter, sni_filter)) { TSError("[%s] Failed to initialize session state with an SNI filter.", traffic_dump::debug_tag); return; } } if (sensitive_fields_were_specified) { - if (!traffic_dump::TransactionData::init(std::move(user_specified_fields))) { + if (!traffic_dump::TransactionData::init(dump_body, std::move(user_specified_fields))) { TSError("[%s] Failed to initialize transaction state with user-specified fields.", traffic_dump::debug_tag); return; } } else { // The user did not provide their own list of sensitive fields. Use the // default. - if (!traffic_dump::TransactionData::init()) { + if (!traffic_dump::TransactionData::init(dump_body)) { TSError("[%s] Failed to initialize transaction state.", traffic_dump::debug_tag); return; } diff --git a/plugins/experimental/traffic_dump/transaction_data.cc b/plugins/experimental/traffic_dump/transaction_data.cc index 51f63507db4..1b2b96a59aa 100644 --- a/plugins/experimental/traffic_dump/transaction_data.cc +++ b/plugins/experimental/traffic_dump/transaction_data.cc @@ -21,15 +21,18 @@ limitations under the License. */ -#include "transaction_data.h" +#include + #include "global_variables.h" #include "json_utils.h" #include "session_data.h" +#include "transaction_data.h" namespace traffic_dump { int TransactionData::transaction_arg_index = 0; sensitive_fields_t TransactionData::sensitive_fields; +bool TransactionData::dump_body = false; std::string default_sensitive_field_value; @@ -44,6 +47,47 @@ sensitive_fields_t default_sensitive_fields = { "Cookie", }; +std::string +TransactionData::request_body_get(TSHttpTxn txnp) +{ + TSDebug(debug_tag, "Getting the HTTP body"); + TSIOBufferReader post_buffer_reader = TSHttpTxnPostBufferReaderGet(txnp); + int64_t read_avail = TSIOBufferReaderAvail(post_buffer_reader); + if (read_avail == 0) { + TSIOBufferReaderFree(post_buffer_reader); + return {}; + } + + // std::string has no reserving allocator. So create an explicitly empty + // string then reserve the bytes for later population. + std::string body_string; + body_string.resize(read_avail); + TSIOBufferReaderCopy(post_buffer_reader, body_string.data(), read_avail); + TSIOBufferReaderFree(post_buffer_reader); + + TSDebug(debug_tag, "Returning %ld body bytes", body_string.size()); + return body_string; +} + +int +TransactionData::request_buffer_handler(TSCont contp, TSEvent event, void *edata) +{ + TSHttpTxn txnp = (TSHttpTxn)(edata); + + if (event == TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE) { + const std::string body = request_body_get(txnp); + TSDebug(debug_tag, "Got a request body of size %zu bytes", body.size()); + + auto *txnData = static_cast(TSUserArgGet(txnp, transaction_arg_index)); + txnData->request_body = body; + TSContDestroy(contp); + } else { + TSDebug(debug_tag, "Request buffer handler received an unrecognized event: %d", event); + } + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); + return 0; +} + void TransactionData::initialize_default_sensitive_field() { @@ -79,17 +123,17 @@ TransactionData::get_sensitive_field_description() } bool -TransactionData::init(sensitive_fields_t &&new_fields) +TransactionData::init(bool dump_body, sensitive_fields_t &&new_fields) { sensitive_fields = std::move(new_fields); - return init_helper(); + return init_helper(dump_body); } bool -TransactionData::init() +TransactionData::init(bool dump_body) { sensitive_fields = default_sensitive_fields; - return init_helper(); + return init_helper(dump_body); } std::string_view @@ -115,6 +159,18 @@ TransactionData::write_content_node(int64_t num_body_bytes) return std::string(R"(,"content":{"encoding":"plain","size":)" + std::to_string(num_body_bytes) + '}'); } +std::string +TransactionData::write_content_node(std::string_view body) +{ + std::ostringstream content_node; + content_node << R"(,"content":{"encoding":"plain","size":)" + std::to_string(body.length()); + if (!body.empty()) { + content_node << R"(,"data":")" + escape_json(std::string(body)) + R"(")"; + } + content_node << '}'; + return content_node.str(); +} + std::string TransactionData::write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_loc) { @@ -216,8 +272,10 @@ TransactionData::remove_scheme_prefix(std::string_view url) } bool -TransactionData::init_helper() +TransactionData::init_helper(bool _dump_body) { + dump_body = _dump_body; + TSDebug(debug_tag, "Dumping body bytes: %s", dump_body ? "true" : "false"); initialize_default_sensitive_field(); const std::string sensitive_fields_string = get_sensitive_field_description(); TSDebug(debug_tag, "Sensitive fields for which generic values will be dumped: %s", sensitive_fields_string.c_str()); @@ -307,6 +365,10 @@ TransactionData::global_transaction_handler(TSCont contp, TSEvent event, void *e txnData->txn_json += R"(,"client-request":{)" + txnData->write_message_node_no_content(buffer, hdr_loc); TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); buffer = nullptr; + if (dump_body) { + TSHttpTxnConfigIntSet(txnp, TS_CONFIG_HTTP_REQUEST_BUFFER_ENABLED, 1); + TSHttpTxnHookAdd(txnp, TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, TSContCreate(request_buffer_handler, TSMutexCreate())); + } } break; } @@ -331,7 +393,11 @@ TransactionData::global_transaction_handler(TSCont contp, TSEvent event, void *e TSMBuffer buffer; TSMLoc hdr_loc; if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &buffer, &hdr_loc)) { - txnData->txn_json += txnData->write_content_node(TSHttpTxnClientReqBodyBytesGet(txnp)) + "}"; + if (dump_body) { + txnData->txn_json += txnData->write_content_node(txnData->request_body) + "}"; + } else { + txnData->txn_json += txnData->write_content_node(TSHttpTxnClientReqBodyBytesGet(txnp)) + "}"; + } TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); buffer = nullptr; } diff --git a/plugins/experimental/traffic_dump/transaction_data.h b/plugins/experimental/traffic_dump/transaction_data.h index b3a13561ae7..a57af011fa2 100644 --- a/plugins/experimental/traffic_dump/transaction_data.h +++ b/plugins/experimental/traffic_dump/transaction_data.h @@ -42,7 +42,10 @@ class TransactionData /** The string for the JSON content of this transaction. */ std::string txn_json; - /** The '"protocol" node for this transaction's server-side connection. */ + /** The string of the client request body, if dump_body is true. */ + std::string request_body; + + /** The "protocol" node for this transaction's server-side conection. */ std::string server_protocol_description; // The index to be used for the TS API for storing this TransactionData on a @@ -53,17 +56,28 @@ class TransactionData /// whose values will be replaced with auto-generated generic content. static sensitive_fields_t sensitive_fields; + /// Whether the user configured the dumping of body content. + static bool dump_body; + public: /** Initialize TransactionData, using the provided sensitive fields. + * + * @param[in] dump_body Whether to dump body content. + * + * @param[in] sensitive_fields_t The HTTP fields considered to have sensitive + * data. * * @return True if initialization is successful, false otherwise. */ - static bool init(sensitive_fields_t &&sensitive_fields); + static bool init(bool dump_body, sensitive_fields_t &&sensitive_fields); /** Initialize TransactionData, using default sensitive fields. + * + * @param[in] dump_body Whether to dump body content. + * * @return True if initialization is successful, false otherwise. */ - static bool init(); + static bool init(bool dump_body); /// Read the txn information from TSMBuffer and write the header information. /// This function does not write the content node. @@ -77,8 +91,14 @@ class TransactionData static int global_transaction_handler(TSCont contp, TSEvent event, void *edata); private: - /** Common logic for the init overloads. */ - static bool init_helper(); + /** Common logic for the init overloads. + * + * @param[in] dump_body Whether the user configured the dumping of body + * content. + * + * @return True if initialization is successful, false otherwise. + */ + static bool init_helper(bool dump_body); /** Initialize the generic sensitive field to be dumped. This is used instead * of the sensitive field values seen on the wire. @@ -91,6 +111,20 @@ class TransactionData */ static std::string get_sensitive_field_description(); + /** Retrieve the request body from the transaction. + * + * @param[in] txnp The transaction from which to retrieve the request body. + * + * @return The request body string. + */ + static std::string request_body_get(TSHttpTxn txnp); + + /** The callback for gathering request body data. + * + * @note This is only called if the user enabled dump_body. + */ + static int request_buffer_handler(TSCont contp, TSEvent event, void *edata); + /** Inspect the field to see whether it is sensitive and return a generic value * of equal size to the original if it is. * @@ -108,6 +142,14 @@ class TransactionData /// "size" std::string write_content_node(int64_t num_body_bytes); + /// Write the content JSON node for an HTTP message. + // + /// "content" + /// "encoding" + /// "size" + /// "data" + std::string write_content_node(std::string_view body); + /** Remove the scheme prefix from the url. * * @return The view without the scheme prefix. diff --git a/proxy/http/HttpDebugNames.cc b/proxy/http/HttpDebugNames.cc index f99e5b1c135..3f691e6546c 100644 --- a/proxy/http/HttpDebugNames.cc +++ b/proxy/http/HttpDebugNames.cc @@ -355,8 +355,8 @@ HttpDebugNames::get_event_name(int event) return "TS_EVENT_VCONN_CLOSE"; case TS_EVENT_LIFECYCLE_MSG: return "TS_EVENT_LIFECYCLE_MSG"; - case TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE: - return "TS_EVENT_HTTP_REQUEST_BUFFER_COMPLETE"; + case TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE: + return "TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE"; case TS_EVENT_MGMT_UPDATE: return "TS_EVENT_MGMT_UPDATE"; case TS_EVENT_INTERNAL_60200: diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index ae4b5a03adf..b374a335e98 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -1616,7 +1616,7 @@ plugins required to work with sni_routing. // void HttpSM::handle_api_return() // // Figures out what to do after calling api callouts -// have finished. This messy and I would like +// have finished. This is messy and I would like // to come up with a cleaner way to handle the api // return. The way we are doing things also makes a // mess of set_next_state() @@ -3619,7 +3619,14 @@ HttpSM::tunnel_handler_post_server(int event, HttpTunnelConsumer *c) { STATE_ENTER(&HttpSM::tunnel_handler_post_server, event); - server_request_body_bytes = c->bytes_written; + // If is_using_post_buffer has been used, then this handler gets called + // twice, once with the buffered request body bytes and a second time with + // the (now) zero length user agent buffer. See wait_for_full_body where + // these bytes are read. Don't clobber the server_request_body_bytes with + // zero on that second read. + if (server_request_body_bytes == 0) { + server_request_body_bytes = c->bytes_written; + } switch (event) { case VC_EVENT_EOS: @@ -5830,9 +5837,16 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type) // Next order of business if copy the remaining data from the // header buffer into new buffer - client_request_body_bytes = post_buffer->write(ua_buffer_reader, chunked ? ua_buffer_reader->read_avail() : post_bytes); + int64_t body_bytes = post_buffer->write(ua_buffer_reader, chunked ? ua_buffer_reader->read_avail() : post_bytes); - ua_buffer_reader->consume(client_request_body_bytes); + // If is_using_post_buffer has been used, then client_request_body_bytes + // will have already been set in wait_for_full_body and there will be + // zero bytes in this user agent buffer. We don't want to clobber + // client_request_body_bytes with a zero value here in those cases. + if (client_request_body_bytes == 0) { + client_request_body_bytes = body_bytes; + } + ua_buffer_reader->consume(body_bytes); p = tunnel.add_producer(ua_entry->vc, post_bytes - transfered_bytes, buf_start, &HttpSM::tunnel_handler_post_ua, HT_HTTP_CLIENT, "user agent post"); } diff --git a/src/traffic_server/InkAPITest.cc b/src/traffic_server/InkAPITest.cc index 2b137453b92..a2dd0f2ba2c 100644 --- a/src/traffic_server/InkAPITest.cc +++ b/src/traffic_server/InkAPITest.cc @@ -83,6 +83,29 @@ #define ERROR_BODY "TESTING ERROR PAGE" #define TRANSFORM_APPEND_STRING "This is a transformed response" +////////////////////////////////////////////////////////////////////////////// +// These static asserts verify the hook vs event value relationship described +// in the doxygen comment for TSHttpHookID. +////////////////////////////////////////////////////////////////////////////// +static_assert(TS_EVENT_HTTP_READ_REQUEST_HDR == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_READ_REQUEST_HDR_HOOK); +static_assert(TS_EVENT_HTTP_OS_DNS == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_OS_DNS_HOOK); +static_assert(TS_EVENT_HTTP_SEND_REQUEST_HDR == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_SEND_REQUEST_HDR_HOOK); +static_assert(TS_EVENT_HTTP_READ_CACHE_HDR == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_READ_CACHE_HDR_HOOK); +static_assert(TS_EVENT_HTTP_READ_RESPONSE_HDR == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_READ_RESPONSE_HDR_HOOK); +static_assert(TS_EVENT_HTTP_SEND_RESPONSE_HDR == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_SEND_RESPONSE_HDR_HOOK); +static_assert(TS_EVENT_HTTP_REQUEST_TRANSFORM == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_REQUEST_TRANSFORM_HOOK); +static_assert(TS_EVENT_HTTP_RESPONSE_TRANSFORM == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_RESPONSE_TRANSFORM_HOOK); +static_assert(TS_EVENT_HTTP_SELECT_ALT == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_SELECT_ALT_HOOK); +static_assert(TS_EVENT_HTTP_TXN_START == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_TXN_START_HOOK); +static_assert(TS_EVENT_HTTP_TXN_CLOSE == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_TXN_CLOSE_HOOK); +static_assert(TS_EVENT_HTTP_SSN_START == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_SSN_START_HOOK); +static_assert(TS_EVENT_HTTP_SSN_CLOSE == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_SSN_CLOSE_HOOK); +static_assert(TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK); +static_assert(TS_EVENT_HTTP_PRE_REMAP == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_PRE_REMAP_HOOK); +static_assert(TS_EVENT_HTTP_POST_REMAP == TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_POST_REMAP_HOOK); +static_assert(TS_EVENT_HTTP_REQUEST_BUFFER_READ_COMPLETE == + TS_EVENT_HTTP_READ_REQUEST_HDR + TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK); + ////////////////////////////////////////////////////////////////////////////// // STRUCTURES ////////////////////////////////////////////////////////////////////////////// @@ -6603,6 +6626,7 @@ enum ORIG_TSHttpHookID { ORIG_TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, ORIG_TS_HTTP_PRE_REMAP_HOOK, ORIG_TS_HTTP_POST_REMAP_HOOK, + ORIG_TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, ORIG_TS_HTTP_RESPONSE_CLIENT_HOOK, ORIG_TS_SSL_FIRST_HOOK, ORIG_TS_VCONN_START_HOOK = ORIG_TS_SSL_FIRST_HOOK, @@ -6616,7 +6640,6 @@ enum ORIG_TSHttpHookID { ORIG_TS_VCONN_OUTBOUND_START_HOOK, ORIG_TS_VCONN_OUTBOUND_CLOSE_HOOK, ORIG_TS_SSL_LAST_HOOK = ORIG_TS_VCONN_OUTBOUND_CLOSE_HOOK, - ORIG_TS_HTTP_REQUEST_BUFFER_READ_COMPLETE_HOOK, ORIG_TS_HTTP_LAST_HOOK }; diff --git a/tests/gold_tests/pluginTest/test_hooks/200.gold b/tests/gold_tests/pluginTest/test_hooks/200.gold new file mode 100644 index 00000000000..a8875735ebb --- /dev/null +++ b/tests/gold_tests/pluginTest/test_hooks/200.gold @@ -0,0 +1,28 @@ +`` +> POST /contentlength HTTP/1.1`` +> Host:`` +> User-Agent: curl/`` +> Accept: */*`` +> Content-Length: 22`` +> Content-Type: application/`` +`` +< HTTP/1.1 200 OK`` +< Server: ATS/`` +< Content-Length: 23`` +< Date:`` +< Age:`` +`` +> POST /chunked`` +> Host:`` +> User-Agent: curl/`` +> Accept: */*`` +> Transfer-Encoding: chunked`` +> Content-Type: application/`` +`` +< HTTP/1.1 200 OK`` +< Server: ATS/`` +< Date: `` +< Age: 0`` +< Transfer-Encoding: chunked`` +< Connection: keep-alive`` +`` diff --git a/tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py b/tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py new file mode 100644 index 00000000000..4d96a76a01a --- /dev/null +++ b/tests/gold_tests/pluginTest/test_hooks/body_buffer.test.py @@ -0,0 +1,128 @@ +''' +Verify HTTP body buffering. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Test.SkipUnless( + Condition.PluginExists('request_buffer.so'), +) + + +def int_to_hex_string(int_value): + ''' + Convert the given int value to a hex string with no '0x' prefix. + + >>> int_to_hex_string(0) + '0' + >>> int_to_hex_string(1) + '1' + >>> int_to_hex_string(10) + 'r' + >>> int_to_hex_string(16) + '0' + >>> int_to_hex_string(17) + 'f1' + ''' + if not isinstance(int_value, int): + raise ValueError("Input should be an int type.") + return hex(int_value).split('x')[1] + + +class BodyBufferTest: + def __init__(cls, description): + Test.Summary = description + cls._origin_max_connections = 3 + cls.setupOriginServer() + cls.setupTS() + + def setupOriginServer(self): + self._server = Test.MakeOriginServer("server") + self.content_length_request_body = "content-length request" + request_header = {"headers": "POST /contentlength HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Content-Length: {}\r\n\r\n".format( + len(self.content_length_request_body)), + "timestamp": "1469733493.993", + "body": self.content_length_request_body} + content_length_response_body = "content-length response" + response_header = {"headers": "HTTP/1.1 200 OK\r\n" + "Server: microserver\r\n" + "Content-Length: {}\r\n\r\n" + "Connection: close\r\n\r\n".format( + len(content_length_response_body)), + "timestamp": "1469733493.993", + "body": content_length_response_body} + self._server.addResponse("sessionlog.json", request_header, response_header) + + self.chunked_request_body = "chunked request" + self.encoded_chunked_request = "{0}\r\n{1}\r\n0\r\n\r\n".format( + int_to_hex_string(len(self.chunked_request_body)), self.chunked_request_body) + request_header2 = {"headers": "POST /chunked HTTP/1.1\r\n" + "Transfer-Encoding: chunked\r\n" + "Host: www.example.com\r\n" + "Connection: keep-alive\r\n\r\n", + "timestamp": "1469733493.993", + "body": self.encoded_chunked_request} + self.chunked_response_body = "chunked response" + self.encoded_chunked_response = "{0}\r\n{1}\r\n0\r\n\r\n".format( + int_to_hex_string(len(self.chunked_response_body)), self.chunked_response_body) + response_header2 = {"headers": "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "Server: microserver\r\n" + "Connection: close\r\n\r\n", + "timestamp": "1469733493.993", + "body": self.encoded_chunked_response} + self._server.addResponse("sessionlog.json", request_header2, response_header2) + + def setupTS(self): + self._ts = Test.MakeATSProcess("ts", select_ports=False) + self._ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(self._server.Variables.Port) + ) + Test.PrepareInstalledPlugin('request_buffer.so', self._ts) + self._ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'request_buffer', + 'proxy.config.http.per_server.connection.max': self._origin_max_connections, + # Disable queueing when connection reaches limit + 'proxy.config.http.per_server.connection.queue_size': 0, + }) + + self._ts.Streams.stderr = Testers.ContainsExpression( + r"request_buffer_plugin gets the request body with length\[{0}\]".format( + len(self.content_length_request_body)), + "Verify that the plugin parsed the content-length request body data.") + self._ts.Streams.stderr += Testers.ContainsExpression( + r"request_buffer_plugin gets the request body with length\[{0}\]".format( + len(self.encoded_chunked_request)), + "Verify that the plugin parsed the chunked request body.") + + def run(self): + tr = Test.AddTestRun() + # Send both a Content-Length request and a chunked-encoded request. + tr.Processes.Default.Command = ( + 'curl -v http://127.0.0.1:{0}/contentlength -d "{1}" --next ' + '-v http://127.0.0.1:{0}/chunked -H "Transfer-Encoding: chunked" -d "{2}"'.format( + self._ts.Variables.port, self.content_length_request_body, self.chunked_request_body)) + tr.Processes.Default.ReturnCode = 0 + tr.Processes.Default.StartBefore(self._server) + tr.Processes.Default.StartBefore(Test.Processes.ts) + tr.Processes.Default.Streams.stderr = "200.gold" + + +bodyBufferTest = BodyBufferTest("Test request body buffering.") +bodyBufferTest.run() diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/200_get.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/200_get.gold new file mode 100644 index 00000000000..614c1392e4b --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/200_get.gold @@ -0,0 +1,12 @@ +`` +> GET /`` HTTP/1.1 +> Host: www.notls.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< HTTP/1.1 200 OK +< Content-Length: 0 +< Date: `` +`` +< Server: ATS/`` +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/200.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/200_get_sensitive_field.gold similarity index 100% rename from tests/gold_tests/pluginTest/traffic_dump/gold/200.gold rename to tests/gold_tests/pluginTest/traffic_dump/gold/200_get_sensitive_field.gold diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/200_post.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/200_post.gold new file mode 100644 index 00000000000..0e5bb5f6fad --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/200_post.gold @@ -0,0 +1,12 @@ +`` +> POST /`` HTTP/1.1 +> Host: www.example.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< HTTP/1.1 200 OK +< Content-Length: `` +< Date: `` +< Age: `` +< Server: ATS/`` +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py index e6e95a1bfa0..d450f7bff66 100644 --- a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py +++ b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py @@ -18,6 +18,7 @@ # limitations under the License. import os + Test.Summary = ''' Verify traffic_dump functionality. ''' @@ -159,6 +160,9 @@ ts.Streams.stderr += Testers.ContainsExpression( "Finish a session with log file of.*bytes", "Verify traffic_dump sees the end of sessions and accounts for it.") +ts.Streams.stderr += Testers.ContainsExpression( + "Dumping body bytes: false", + "Verify that dumping body bytes is enabled.") # Set up the json replay file expectations. replay_file_session_1 = os.path.join(replay_dir, "127", "0000000000000000") @@ -196,7 +200,7 @@ '-H"Host: www.notls.com" -H"X-Request-1: ultra_sensitive" --verbose'.format( ts.Variables.port)) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Streams.stderr = "gold/200.gold" +tr.Processes.Default.Streams.stderr = "gold/200_get_sensitive_field.gold" tr.StillRunningAfter = server tr.StillRunningAfter = ts http_protocols = "tcp,ip" @@ -208,7 +212,7 @@ '-H"X-Request-2: also_very_sensitive" --verbose'.format( ts.Variables.port)) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Streams.stderr = "gold/200.gold" +tr.Processes.Default.Streams.stderr = "gold/200_get_sensitive_field.gold" tr.StillRunningAfter = server tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_ip_filter.test.py b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_ip_filter.test.py new file mode 100644 index 00000000000..f5cdafdcb57 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_ip_filter.test.py @@ -0,0 +1,150 @@ +""" +Verify traffic_dump IP filter functionality. +""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Verify traffic_dump IP filter functionality. +''' + +Test.SkipUnless( + Condition.PluginExists('traffic_dump.so'), +) + +# Configure the origin server. +server = Test.MakeOriginServer("server", both=True) + +request_header = {"headers": "GET /empty HTTP/1.1\r\n" + "Host: www.notls.com\r\n" + "Content-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Content-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) + + +def get_common_ats_process(name, plugin_command, replay_exists): + """ + Get an ATS process with some common configuration. + + These tests have different log expectations, but have generally the same + ATS Process configuration. This function returns a Process with that common + configuration. + + Returns: + A configured ATS Process. + The replay file. + """ + ts = Test.MakeATSProcess(name) + replay_dir = os.path.join(ts.RunDirectory, name, "log") + ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'traffic_dump', + }) + ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) + ) + # Configure traffic_dump as specified. + ts.Disk.plugin_config.AddLine(plugin_command.format(replay_dir)) + + ts_replay_file_session_1 = os.path.join(replay_dir, "127", "0000000000000000") + ts.Disk.File(ts_replay_file_session_1, exists=replay_exists) + return ts, ts_replay_file_session_1 + + +# +# Test 1: Verify -4 works for a specified address. +# +tr = Test.AddTestRun("Verify that -4 matches 127.0.0.1 as expected") +ts1, ts1_replay_file = get_common_ats_process( + "ts1", + 'traffic_dump.so --logdir {0} --sample 1 --limit 1000000000 -4 127.0.0.1', + replay_exists=True) +ts1.Streams.stderr += Testers.ContainsExpression( + "Filtering to only dump connections with ip: 127.0.0.1", + "Verify the IP filter status message.") + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts1) +tr.Processes.Default.Command = \ + ('curl http://127.0.0.1:{0}/empty -H"Host: www.notls.com" --verbose'.format( + ts1.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_get.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts1 + +# Verify that the expected request body was recorded. +tr = Test.AddTestRun("Verify that the expected request body was recorded.") +verify_replay = "verify_replay.py" +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = "python3 {0} {1} {2}".format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + ts1_replay_file) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts1 + +# +# Test 2: Verify -4 filters out other addresses. +# +tr = Test.AddTestRun("Verify that -4 filters out our non-matching IP as expected") +ts2, ts2_replay_file = get_common_ats_process( + "ts2", + 'traffic_dump.so --logdir {0} --sample 1 --limit 1000000000 -4 1.2.3.4', + replay_exists=False) +ts2.Streams.stderr += Testers.ContainsExpression( + "Filtering to only dump connections with ip: 1.2.3.4", + "Verify the IP filter status message.") + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts2) +tr.Processes.Default.Command = \ + ('curl http://127.0.0.1:{0}/empty -H"Host: www.notls.com" --verbose'.format( + ts2.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_get.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts2 + +# +# Test 3: Verify -4 recognizes an invalid IP address string. +# +tr = Test.AddTestRun("Verify that -4 detects an invalid IP string") +invalid_ip = "this_is_not_a_valid_ip_string" +ts3, ts3_replay_file = get_common_ats_process( + "ts3", + 'traffic_dump.so --logdir {0} --sample 1 --limit 1000000000 -4 ' + invalid_ip, + replay_exists=False) +ts3.Disk.diags_log.Content = Testers.ContainsExpression( + "Problems parsing IP filter address argument: {}".format(invalid_ip), + "Verify traffic_dump detects an invalid IPv4 address.") + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts3) +tr.Processes.Default.Command = \ + ('curl http://127.0.0.1:{0}/empty -H"Host: www.notls.com" --verbose'.format( + ts3.Variables.port)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_get.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts3 diff --git a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_request_body.test.py b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_request_body.test.py new file mode 100644 index 00000000000..bcbb9e7781b --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump_request_body.test.py @@ -0,0 +1,182 @@ +""" +Verify traffic_dump request body functionality. +""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +Test.Summary = ''' +Verify traffic_dump request body functionality. +''' + +Test.SkipUnless( + Condition.PluginExists('traffic_dump.so'), +) + +# Configure the origin server. +server = Test.MakeOriginServer("server", both=True) + +request_header = {"headers": "GET /empty HTTP/1.1\r\n" + "Host: www.notls.com\r\n" + "Content-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +response_header = {"headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Content-Length: 0\r\n\r\n", + "timestamp": "1469733493.993", "body": ""} +server.addResponse("sessionfile.log", request_header, response_header) +request_header = {"headers": "POST /with_content_length HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Content-Length: 4\r\n\r\n", + "timestamp": "1469733493.993", "body": "1234"} +response_header = {"headers": "HTTP/1.1 200 OK\r\n" + "Connection: close\r\n" + "Content-Length: 4\r\n\r\n", + "timestamp": "1469733493.993", "body": "1234"} +server.addResponse("sessionfile.log", request_header, response_header) + + +def get_common_ats_process(name): + """ + Get an ATS process with some common configuration. + + These tests have different log expectations, but have generally the same + ATS Process configuration. This function returns a Process with that common + configuration. + + Returns: + A configured ATS Process. + The replay file. + """ + ts = Test.MakeATSProcess(name) + replay_dir = os.path.join(ts.RunDirectory, name, "log") + ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'traffic_dump', + }) + ts.Disk.remap_config.AddLine( + 'map / http://127.0.0.1:{0}'.format(server.Variables.Port) + ) + # Configure traffic_dump to dump body bytes (-b). + ts.Disk.plugin_config.AddLine( + 'traffic_dump.so --logdir {0} --sample 1 --limit 1000000000 -b'.format(replay_dir) + ) + + ts_replay_file_session_1 = os.path.join(replay_dir, "127", "0000000000000000") + ts.Disk.File(ts_replay_file_session_1, exists=True) + + ts.Streams.stderr = Testers.ContainsExpression( + "Dumping body bytes: true", + "Verify that dumping body bytes is enabled.") + return ts, ts_replay_file_session_1 + + +# +# Test 1: Verify a request without a body is dumped correctly. +# +tr = Test.AddTestRun("An empty request body is handled correctly.") +ts1, ts1_replay_file = get_common_ats_process("ts1") + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts1) +tr.Processes.Default.Command = 'curl http://127.0.0.1:{0}/empty -H"Host: www.notls.com" --verbose'.format( + ts1.Variables.port) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_get.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts1 + +tr = Test.AddTestRun("Verify the json content of the first session") +verify_replay = "verify_replay.py" +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = 'python3 {0} {1} {2}'.format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + ts1_replay_file) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts1 + +# +# Test 2: Verify request body can be dumped. +# +tr = Test.AddTestRun("Verify body bytes can be dumped") +ts2, ts2_replay_file = get_common_ats_process("ts2") +ts2.Streams.stderr += Testers.ContainsExpression( + "Got a request body of size 4 bytes", + "Verify logging of the dumped body bytes.") + +request_body = "1234" + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts2) +tr.Processes.Default.Command = \ + ('curl http://127.0.0.1:{0}/with_content_length -H"Host: www.example.com" ' + '--verbose -d "{1}"'.format( + ts2.Variables.port, + request_body)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_post.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts2 + +# Verify that the expected request body was recorded. +tr = Test.AddTestRun("Verify that the expected request body was recorded.") +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = "python3 {0} {1} {2} --request_body {3}".format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + ts2_replay_file, + request_body) +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts2 + +# +# Test 3: Verify request body bytes are escaped. +# +tr = Test.AddTestRun("Verify body bytes are dumped") +ts3, ts3_replay_file = get_common_ats_process("ts3") +ts3.Streams.stderr += Testers.ContainsExpression( + "Got a request body of size 5 bytes", + "Verify logging of the dumped body bytes.") + +request_body = '12"34' + +tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port)) +tr.Processes.Default.StartBefore(ts3) +tr.Processes.Default.Command = \ + ("curl http://127.0.0.1:{0}/with_content_length -H'Host: www.example.com' " + "--verbose -d '{1}'".format( + ts3.Variables.port, + request_body)) +tr.Processes.Default.ReturnCode = 0 +tr.Processes.Default.Streams.stderr = "gold/200_post.gold" +tr.StillRunningAfter = server +tr.StillRunningAfter = ts3 + +# Verify that the expected request body was recorded. +tr = Test.AddTestRun("Verify that the expected request body was recorded.") +tr.Setup.CopyAs(verify_replay, Test.RunDirectory) +tr.Processes.Default.Command = "python3 {0} {1} {2} --request_body '{3}'".format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + ts3_replay_file, + r'12"34') +tr.Processes.Default.ReturnCode = 0 +tr.StillRunningAfter = server +tr.StillRunningAfter = ts3 diff --git a/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py b/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py index a741e810f37..74851188942 100644 --- a/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py +++ b/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py @@ -254,11 +254,42 @@ def verify_server_tls_features(replay_json, expected_tls_features): return True +def verify_client_request_body_bytes(replay_json, expected_body_bytes): + """ + Verify that the replay file has the specified body bytes in it. + """ + try: + received_body_bytes = replay_json['sessions'][0]['transactions'][0]['client-request']['content']['data'] + except KeyError: + print("The replay file did not have a body element in the first transaction.") + return False + + if received_body_bytes != expected_body_bytes: + print("Expected body bytes of '{0}' but got '{1}'".format(expected_body_bytes, received_body_bytes)) + return False + + # If the client request had that many bytes, verify that the proxy-request + # does too. This is not guaranteed to always be true, but for our autest it + # currently holds and this check is worthwhile. + try: + proxy_request_body_size = replay_json['sessions'][0]['transactions'][0]['proxy-request']['content']['size'] + except KeyError: + print("The replay file did not have a proxgy-rerquest content size element in the first transaction.") + return False + + if int(proxy_request_body_size) != len(expected_body_bytes): + print("Expected the proxy-request content size to be '{0}' but got '{1}'".format( + len(expected_body_bytes), proxy_request_body_size)) + return False + + return True + + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("schema_file", type=argparse.FileType('r'), - help="The schema in which to interpret validate the replay file.") + help="The schema in which to validate the replay file.") parser.add_argument("replay_file", type=argparse.FileType('r'), help="The replay file to validate.") @@ -278,6 +309,9 @@ def parse_args(): help="The TLS values to expect for the client connection.") parser.add_argument("--server-tls-features", help="The TLS values to expect for the server connection.") + parser.add_argument("--request_body", + type=str, + help="Verify that the client request has the specified body bytes.") return parser.parse_args() @@ -323,6 +357,9 @@ def main(): if args.server_tls_features and not verify_server_tls_features(replay_json, args.server_tls_features): return 1 + if args.request_body and not verify_client_request_body_bytes(replay_json, args.request_body): + return 1 + return 0