diff --git a/doc/developer-guide/api/functions/TSHttpTxnClientStreamIdGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnClientStreamIdGet.en.rst new file mode 100644 index 00000000000..69016bbe8a8 --- /dev/null +++ b/doc/developer-guide/api/functions/TSHttpTxnClientStreamIdGet.en.rst @@ -0,0 +1,49 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSHttpTxnClientStreamIdGet +************************** + +Synopsis +======== + +.. code-block:: cpp + + #include + +.. function:: TSReturnCode TSHttpTxnClientStreamIdGet(TSHttpTxn txnp, uint64_t* stream_id) + +Description +=========== + +Retrieve the stream identification for the HTTP stream of which the provided +transaction is a part. The resultant stream identifier is populated in the +``stream_id`` output parameter. + +This interface currently only supports HTTP/2 streams. See RFC 7540 section +5.1.1 for details concerning HTTP/2 stream identifiers. + +This API returns an error if the provided transaction is not an HTTP/2 +transaction. + +See Also +======== + +:doc:`TSHttpTxnClientStreamPriorityGet.en` diff --git a/doc/developer-guide/api/functions/TSHttpTxnClientStreamPriorityGet.en.rst b/doc/developer-guide/api/functions/TSHttpTxnClientStreamPriorityGet.en.rst new file mode 100644 index 00000000000..ee5b26ada51 --- /dev/null +++ b/doc/developer-guide/api/functions/TSHttpTxnClientStreamPriorityGet.en.rst @@ -0,0 +1,67 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed + with this work for additional information regarding copyright + ownership. The ASF licenses this file to you under the Apache + License, Version 2.0 (the "License"); you may not use this file + except in compliance with the License. You may obtain a copy of + the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied. See the License for the specific language governing + permissions and limitations under the License. + +.. include:: ../../../common.defs + +.. default-domain:: c + +TSHttpTxnClientStreamPriorityGet +******************************** + +Synopsis +======== + +.. code-block:: cpp + + #include + +.. function:: TSReturnCode TSHttpTxnClientStreamPriorityGet(TSHttpTxn txnp, TSHttpPriority* priority) + +Description +=========== + +Retrieve the priority information for the HTTP stream associated with the +provided transaction. The resultant priority information is populated in the +``priority`` output variable. The ``TSHttpPriority`` type is designed to be +agnostic of the various HTTP protocol versions that support HTTP streams. The +user should pass a pointer casted to ``TSHttpPriority`` from a previously +allocated ``TSHttp2Priority`` structure. This design anticipates future support +for HTTP versions that support streams, such as HTTP/3. + +The ``TSHttp2Priority`` structure has the following declaration: + +.. code-block:: cpp + + typedef struct { + uint8_t priority_type; /** HTTP_PROTOCOL_TYPE_HTTP_2 */ + int32_t stream_dependency; + uint8_t weight; + } TSHttp2Priority; + +In a call to ``TSHttpTxnClientStreamPriorityGet``, the dependency and weight +will be populated in the ``stream_dependency`` and ``weight`` members, +respectively. If the stream associated with the given transaction has no +dependency, then the ``stream_dependency`` output parameter will be populated +with ``-1`` and the value of ``weight`` will be meaningless. See RFC 7540 +section 5.3 for details concerning HTTP/2 stream priority. + +This API returns an error if the provided transaction is not an HTTP/2 +transaction. + +See Also +======== + +:doc:`TSHttpTxnClientStreamIdGet.en` diff --git a/include/ts/apidefs.h.in b/include/ts/apidefs.h.in index 57073351c10..27e1522dd91 100644 --- a/include/ts/apidefs.h.in +++ b/include/ts/apidefs.h.in @@ -873,6 +873,44 @@ typedef enum { TS_USER_ARGS_COUNT ///< Fake enum, # of valid entries. } TSUserArgType; +/** An enumeration of HTTP version types for the priority functions that behave + * differently across HTTP protocols. */ +typedef enum { + HTTP_PRIORITY_TYPE_HTTP_UNSPECIFIED = 1, + HTTP_PRIORITY_TYPE_HTTP_2, + HTTP_PRIORITY_TYPE_HTTP_3, +} TSHttpPriorityType; + +/** The abstract type of the various HTTP priority implementations. */ +typedef struct { + /** The reference to the concrete HTTP priority implementation. This will be + * a value from TSHttpPriorityType. */ + uint8_t priority_type; + /** The space allocated for the concrete priority implementation. + * + * Note that this has to take padding into account. There is a static_assert + * in InkAPI.cc to verify that TSHttpPriority is at least as large as + * TSHttp2Priority. As other structures are added that are represented by + * TSHttpPriority add more static_asserts to verify that TSHttpPriority is as + * large as it needs to be. + */ + uint8_t data[7]; +} TSHttpPriority; + +/** A structure for HTTP/2 priority. + * + * For an explanation of these terms with respect to HTTP/2, see RFC 7540, + * section 5.3. + */ +typedef struct { + uint8_t priority_type; /** HTTP_PROTOCOL_TYPE_HTTP_2 */ + uint8_t weight; + /** The stream dependency. Per spec, see RFC 7540 section 6.2, this is 31 + * bits. We use a signed 32 bit stucture to store either a valid dependency + * or -1 if the stream has no dependency. */ + int32_t stream_dependency; +} TSHttp2Priority; + typedef struct tsapi_file *TSFile; typedef struct tsapi_mloc *TSMLoc; diff --git a/include/ts/ts.h b/include/ts/ts.h index 2df875b4058..1f133ba5960 100644 --- a/include/ts/ts.h +++ b/include/ts/ts.h @@ -2574,6 +2574,32 @@ tsapi TSIOBufferReader TSHttpTxnPostBufferReaderGet(TSHttpTxn txnp); */ tsapi TSReturnCode TSHttpTxnServerPush(TSHttpTxn txnp, const char *url, int url_len); +/** Retrieve the client side stream id for the stream of which the + * provided transaction is a part. + * + * @param[in] txnp The Transaction for which the stream id should be retrieved. + * @param[out] stream_id The stream id for this transaction. + * + * @return TS_ERROR if a stream id cannot be retrieved for the given + * transaction given its protocol. For instance, if txnp is an HTTP/1.1 + * transaction, then a TS_ERROR will be returned because HTTP/1.1 does not + * implement streams. + */ +tsapi TSReturnCode TSHttpTxnClientStreamIdGet(TSHttpTxn txnp, uint64_t *stream_id); + +/** Retrieve the client side priority for the stream of which the + * provided transaction is a part. + * + * @param[in] txnp The Transaction for which the stream id should be retrieved. + * @param[out] priority The priority for the stream in this transaction. + * + * @return TS_ERROR if a priority cannot be retrieved for the given + * transaction given its protocol. For instance, if txnp is an HTTP/1.1 + * transaction, then a TS_ERROR will be returned because HTTP/1.1 does not + * implement stream priorities. + */ +tsapi TSReturnCode TSHttpTxnClientStreamPriorityGet(TSHttpTxn txnp, TSHttpPriority *priority); + #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/plugins/experimental/traffic_dump/session_data.cc b/plugins/experimental/traffic_dump/session_data.cc index de41e09dc6a..dac9eaeae85 100644 --- a/plugins/experimental/traffic_dump/session_data.cc +++ b/plugins/experimental/traffic_dump/session_data.cc @@ -46,27 +46,32 @@ char const constexpr *const json_closing = "]}]}"; /** * A mapping from IP_PROTO_TAG to the string describing the JSON protocol node. */ -std::unordered_map tag_to_node = { - {std::string(IP_PROTO_TAG_IPV4), R"("name":"ip","version":"4")"}, - {std::string(IP_PROTO_TAG_IPV6), R"("name":"ip","version":"6")"}, +std::unordered_map tag_to_node = { + {IP_PROTO_TAG_IPV4, R"("name":"ip","version":"4")"}, + {IP_PROTO_TAG_IPV6, R"("name":"ip","version":"6")"}, - {std::string(IP_PROTO_TAG_TCP), R"("name":"tcp")"}, - {std::string(IP_PROTO_TAG_UDP), R"("name":"udp")"}, + {IP_PROTO_TAG_TCP, R"("name":"tcp")"}, + {IP_PROTO_TAG_UDP, R"("name":"udp")"}, - {std::string(IP_PROTO_TAG_QUIC), R"("name:":"quic")"}, + {IP_PROTO_TAG_QUIC, R"("name:":"quic")"}, - {std::string(IP_PROTO_TAG_TLS_1_0), R"("name":"tls","version":"1.0")"}, - {std::string(IP_PROTO_TAG_TLS_1_1), R"("name":"tls","version":"1.1")"}, - {std::string(IP_PROTO_TAG_TLS_1_2), R"("name":"tls","version":"1.2")"}, - {std::string(IP_PROTO_TAG_TLS_1_3), R"("name":"tls","version":"1.3")"}, + {IP_PROTO_TAG_TLS_1_0, R"("name":"tls","version":"1.0")"}, + {IP_PROTO_TAG_TLS_1_1, R"("name":"tls","version":"1.1")"}, + {IP_PROTO_TAG_TLS_1_2, R"("name":"tls","version":"1.2")"}, + {IP_PROTO_TAG_TLS_1_3, R"("name":"tls","version":"1.3")"}, - {std::string(IP_PROTO_TAG_HTTP_0_9), R"("name":"http","version":"0.9")"}, - {std::string(IP_PROTO_TAG_HTTP_1_0), R"("name":"http","version":"1.0")"}, - {std::string(IP_PROTO_TAG_HTTP_1_1), R"("name":"http","version":"1.1")"}, - {std::string(IP_PROTO_TAG_HTTP_2_0), R"("name":"http","version":"2")"}, + {IP_PROTO_TAG_HTTP_0_9, R"("name":"http","version":"0.9")"}, + {IP_PROTO_TAG_HTTP_1_0, R"("name":"http","version":"1.0")"}, + {IP_PROTO_TAG_HTTP_1_1, R"("name":"http","version":"1.1")"}, + {IP_PROTO_TAG_HTTP_2_0, R"("name":"http","version":"2")"}, - {std::string(IP_PROTO_TAG_HTTP_QUIC), R"("name":"http","version":"0.9")"}, - {std::string(IP_PROTO_TAG_HTTP_3), R"("name":"http","version":"3")"}, + {IP_PROTO_TAG_HTTP_QUIC, R"("name":"http","version":"0.9")"}, + {IP_PROTO_TAG_HTTP_3, R"("name":"http","version":"3")"}, +}; + +std::unordered_map http_tag_to_version = { + {IP_PROTO_TAG_HTTP_0_9, "0.9"}, {IP_PROTO_TAG_HTTP_1_0, "1.0"}, {IP_PROTO_TAG_HTTP_1_1, "1.1"}, + {IP_PROTO_TAG_HTTP_2_0, "2"}, {IP_PROTO_TAG_HTTP_QUIC, "0.9"}, {IP_PROTO_TAG_HTTP_3, "3"}, }; /** Create a TLS characteristics node. @@ -137,53 +142,6 @@ get_server_tls_description(TSHttpTxn txnp) TSVConn server_ssn_vc = TSHttpTxnServerVConnGet(txnp); return get_tls_description_helper(server_ssn_vc); } - -using get_protocol_stack_f = std::function; -using get_tls_description_f = std::function; - -/** Create the protocol stack for a session. - * - * This function encapsulates the logic common between the client-side and - * server-side logic for populating a protocol stack. - * - * @param[in] get_protocol_stack The function to use to populate a protocol - * stack. - * - * @return The description of the protocol stack. - */ -std::string -get_protocol_stack_helper(const get_protocol_stack_f &get_protocol_stack, const get_tls_description_f &get_tls_node) -{ - std::ostringstream protocol_description; - protocol_description << R"("protocol":[)"; - char const *protocol[10]; - int count = -1; - TSAssert(TS_SUCCESS == get_protocol_stack(10, protocol, &count)); - bool is_first_printed_protocol = true; - for (int i = 0; i < count; ++i) { - std::string_view protocol_string(protocol[i]); - if (!is_first_printed_protocol) { - protocol_description << ","; - } - is_first_printed_protocol = false; - if (protocol_string.find("tls") != std::string::npos) { - protocol_description << '{' << get_tls_node() << '}'; - } else { - auto search = tag_to_node.find(std::string(protocol_string)); - if (search == tag_to_node.end()) { - // If the tag from get_protocol_stack is not in our list, then our - // tag_to_node has not been updated with the new tag. Update tag_to_node. - TSError("[%s] Missing tag node description: '%.*s'", traffic_dump::debug_tag, static_cast(protocol_string.length()), - protocol_string.data()); - protocol_description << R"({"name":")" << protocol_string << R"("})"; - } else { - protocol_description << '{' << search->second << '}'; - } - } - } - protocol_description << "]"; // Close the "protocol" sequence. - return protocol_description.str(); -} } // namespace namespace traffic_dump @@ -254,6 +212,46 @@ SessionData::init(std::string_view log_directory, int64_t max_disk_usage, int64_ return true; } +std::string +SessionData::get_protocol_stack_helper(const get_protocol_stack_f &get_protocol_stack, const get_tls_description_f &get_tls_node) +{ + std::ostringstream protocol_description; + protocol_description << R"("protocol":[)"; + char const *protocol[10]; + int count = -1; + TSAssert(TS_SUCCESS == get_protocol_stack(10, protocol, &count)); + bool is_first_printed_protocol = true; + for (int i = 0; i < count; ++i) { + std::string_view protocol_string(protocol[i]); + if (!is_first_printed_protocol) { + protocol_description << ","; + } + is_first_printed_protocol = false; + if (protocol_string.find("tls") != std::string::npos) { + protocol_description << '{' << get_tls_node() << '}'; + } else { + auto search = tag_to_node.find(std::string(protocol_string)); + if (search == tag_to_node.end()) { + // If the tag from get_protocol_stack is not in our list, then our + // tag_to_node has not been updated with the new tag. Update tag_to_node. + TSError("[%s] Missing tag node description: '%.*s'", traffic_dump::debug_tag, static_cast(protocol_string.length()), + protocol_string.data()); + protocol_description << R"({"name":")" << protocol_string << R"("})"; + } else { + protocol_description << '{' << search->second << '}'; + } + + // See whether an HTTP version is provided. If so, record it. + auto const it = http_tag_to_version.find(std::string(protocol_string)); + if (it != http_tag_to_version.end()) { + this->http_version_in_client_stack = it->second; + } + } + } + protocol_description << "]"; // Close the "protocol" sequence. + return protocol_description.str(); +} + std::string SessionData::get_client_protocol_description(TSHttpSsn client_ssnp) { @@ -342,6 +340,12 @@ SessionData::write_transaction_to_disk(std::string_view content) return result; } +std::string +SessionData::get_http_version_in_client_stack() const +{ + return http_version_in_client_stack; +} + int SessionData::session_aio_handler(TSCont contp, TSEvent event, void *edata) { @@ -437,8 +441,7 @@ SessionData::global_session_handler(TSCont contp, TSEvent event, void *edata) // "protocol": // This is the protocol stack for the client side of the session. - std::string protocol_description = get_client_protocol_description(ssnp); - + std::string protocol_description = ssnData->get_client_protocol_description(ssnp); std::string beginning = R"({"meta":{"version":"1.0"},"sessions":[{)" + protocol_description + R"(,"connection-time":)" + std::to_string(start.count()) + R"(,"transactions":[)"; diff --git a/plugins/experimental/traffic_dump/session_data.h b/plugins/experimental/traffic_dump/session_data.h index b25a42f57bb..68c865039c9 100644 --- a/plugins/experimental/traffic_dump/session_data.h +++ b/plugins/experimental/traffic_dump/session_data.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "ts/ts.h" @@ -66,6 +67,9 @@ class SessionData ts::file::path log_name; /// Whether the first transaction in this session has been written. bool has_written_first_transaction = false; + /// The HTTP version specified in the client protocol stack, or empty string + /// if it was not specified. + std::string http_version_in_client_stack; TSCont aio_cont = nullptr; /// AIO continuation callback TSCont txn_cont = nullptr; /// Transaction continuation callback @@ -141,7 +145,7 @@ class SessionData * @return A JSON description of the server protocol stack for the * transaction. */ - static std::string get_server_protocol_description(TSHttpTxn txnp); + std::string get_server_protocol_description(TSHttpTxn txnp); /** Write the string to the session's dump file. * @@ -161,6 +165,19 @@ class SessionData */ int write_transaction_to_disk(std::string_view content); + /** The HTTP version specified in the client-side protocol stack. + * + * The client protocol stack is obtained at session negotiation, before HTTP + * traffic is passed. So it may contain stack information if it was + * negotiated in the TLS handshake, as is often the case with HTTP/2, but it + * may not. This function returns whether the protocol stack contained HTTP + * information or not. + * + * @return The HTTP version in the client stack or empty string if it was not + * specified. + */ + std::string get_http_version_in_client_stack() const; + private: /** Write the string to the session's dump file. * @@ -173,13 +190,30 @@ class SessionData */ int write_to_disk_no_lock(std::string_view content); + using get_protocol_stack_f = std::function; + using get_tls_description_f = std::function; + + /** Create the protocol stack for a session. + * + * This function encapsulates the logic common between the client-side and + * server-side logic for populating a protocol stack. + * + * @param[in] get_protocol_stack The function to use to populate a protocol + * stack. + * + * @return The description of the protocol stack and True if the stack + * contained an HTTP description, false otherwise. + */ + std::string get_protocol_stack_helper(const get_protocol_stack_f &get_protocol_stack, const get_tls_description_f &get_tls_node); + /** Get the JSON string that describes the client session stack. * * @param[in] ssnp The reference to the client session. * - * @return A JSON description of the client protocol stack. + * @return A description of the client protocol stack and True if the stack + * contained an HTTP description, false otherwise. */ - static std::string get_client_protocol_description(TSHttpSsn ssnp); + std::string get_client_protocol_description(TSHttpSsn ssnp); /** The handler callback for when async IO is done. Used for cleanup. */ static int session_aio_handler(TSCont contp, TSEvent event, void *edata); diff --git a/plugins/experimental/traffic_dump/transaction_data.cc b/plugins/experimental/traffic_dump/transaction_data.cc index 51f63507db4..684cc89d1e0 100644 --- a/plugins/experimental/traffic_dump/transaction_data.cc +++ b/plugins/experimental/traffic_dump/transaction_data.cc @@ -21,6 +21,8 @@ limitations under the License. */ +#include + #include "transaction_data.h" #include "global_variables.h" #include "json_utils.h" @@ -78,6 +80,11 @@ TransactionData::get_sensitive_field_description() return sensitive_fields_string; } +TransactionData::TransactionData(TSHttpTxn txnp, std::string_view http_version_from_client_stack) + : _txnp{txnp}, _http_version_from_client_stack{http_version_from_client_stack} +{ +} + bool TransactionData::init(sensitive_fields_t &&new_fields) { @@ -116,7 +123,7 @@ TransactionData::write_content_node(int64_t num_body_bytes) } std::string -TransactionData::write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_loc) +TransactionData::write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_loc, std::string_view http_version) { std::string result; int len = 0; @@ -124,10 +131,14 @@ TransactionData::write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_lo TSMLoc url_loc = nullptr; // 1. "version" - // Note that we print this for both requests and responses, so the first - // element in each has to start with a comma. - int version = TSHttpHdrVersionGet(buffer, hdr_loc); - result += R"("version":")" + std::to_string(TS_HTTP_MAJOR(version)) + "." + std::to_string(TS_HTTP_MINOR(version)) + '"'; + result += R"("version":")"; + if (http_version.empty()) { + int version = TSHttpHdrVersionGet(buffer, hdr_loc); + result += std::to_string(TS_HTTP_MAJOR(version)) + "." + std::to_string(TS_HTTP_MINOR(version)); + } else { + result += http_version; + } + result += R"(",)"; // Log scheme+method+request-target or status+reason based on header type if (TSHttpHdrTypeGet(buffer, hdr_loc) == TS_HTTP_TYPE_REQUEST) { @@ -135,7 +146,7 @@ TransactionData::write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_lo // 2. "scheme": cp = TSUrlSchemeGet(buffer, url_loc, &len); TSDebug(debug_tag, "write_message_node(): found scheme %.*s ", len, cp); - result += "," + traffic_dump::json_entry("scheme", cp, len); + result += traffic_dump::json_entry("scheme", cp, len); // 3. "method":(string) cp = TSHttpHdrMethodGet(buffer, hdr_loc, &len); @@ -159,15 +170,15 @@ TransactionData::write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_lo } TSDebug(debug_tag, "write_message_node(): found host target %.*s", static_cast(url_string.size()), url_string.data()); - result += "," + traffic_dump::json_entry("url", url_string); + result += ',' + traffic_dump::json_entry("url", url_string); TSfree(url); TSHandleMLocRelease(buffer, hdr_loc, url_loc); } else { // 2. "status":(string) - result += R"(,"status":)" + std::to_string(TSHttpHdrStatusGet(buffer, hdr_loc)); + result += R"("status":)" + std::to_string(TSHttpHdrStatusGet(buffer, hdr_loc)); // 3. "reason":(string) cp = TSHttpHdrReasonGet(buffer, hdr_loc, &len); - result += "," + traffic_dump::json_entry("reason", cp, len); + result += ',' + traffic_dump::json_entry("reason", cp, len); } // "headers": [[name(string), value(string)]] @@ -197,9 +208,9 @@ TransactionData::write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_lo } std::string -TransactionData::write_message_node(TSMBuffer &buffer, TSMLoc &hdr_loc, int64_t num_body_bytes) +TransactionData::write_message_node(TSMBuffer &buffer, TSMLoc &hdr_loc, int64_t num_body_bytes, std::string_view http_version) { - std::string result = write_message_node_no_content(buffer, hdr_loc); + std::string result = write_message_node_no_content(buffer, hdr_loc, http_version); result += write_content_node(num_body_bytes); return result + "}"; } @@ -237,6 +248,64 @@ TransactionData::init_helper() return true; } +void +TransactionData::write_client_request_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_loc) +{ + std::ostringstream client_request_node; + client_request_node << R"(,"client-request":{)"; + + auto const http_version = _http_version_from_client_stack; + if (http_version == "2") { + client_request_node << R"("http2":{)"; + + uint64_t stream_id; + TSAssert(TS_SUCCESS == TSHttpTxnClientStreamIdGet(_txnp, &stream_id)); + client_request_node << R"("stream-id":)" << std::to_string(stream_id); + + TSHttp2Priority priority; + memset(&priority, 0, sizeof(priority)); + TSAssert(TS_SUCCESS == TSHttpTxnClientStreamPriorityGet(_txnp, reinterpret_cast(&priority))); + TSAssert(HTTP_PRIORITY_TYPE_HTTP_2 == priority.priority_type); + // Http2Stream uses -1 as an indication that no priority was set. + if (priority.stream_dependency != -1) { + client_request_node << R"(,"priority":{)"; + client_request_node << R"("stream-depenency":)" << std::to_string(priority.stream_dependency); + client_request_node << R"(,"weight":)" << std::to_string(priority.weight); + client_request_node << "}"; + } + + client_request_node << "},"; + } + + // We don't have an accurate view of the body size until TXN_CLOSE so we hold + // off on writing the content:size node until then. + client_request_node << write_message_node_no_content(buffer, hdr_loc, http_version); + _txn_json += client_request_node.str(); +} + +void +TransactionData::write_proxy_request_node(TSMBuffer &buffer, TSMLoc &hdr_loc) +{ + std::ostringstream proxy_request_node; + proxy_request_node << R"(,"proxy-request":{)"; + proxy_request_node << _server_protocol_description + ","; + proxy_request_node << write_message_node(buffer, hdr_loc, TSHttpTxnServerReqBodyBytesGet(_txnp)); + _txn_json += proxy_request_node.str(); +} + +void +TransactionData::write_server_response_node(TSMBuffer &buffer, TSMLoc &hdr_loc) +{ + _txn_json += R"(,"server-response":{)" + write_message_node(buffer, hdr_loc, TSHttpTxnServerRespBodyBytesGet(_txnp)); +} + +void +TransactionData::write_proxy_response_node(TSMBuffer &buffer, TSMLoc &hdr_loc) +{ + _txn_json += R"(,"proxy-response":{)" + + write_message_node(buffer, hdr_loc, TSHttpTxnClientRespBodyBytesGet(_txnp), _http_version_from_client_stack); +} + // Transaction handler: writes headers to the log file using AIO int TransactionData::global_transaction_handler(TSCont contp, TSEvent event, void *edata) @@ -262,7 +331,7 @@ TransactionData::global_transaction_handler(TSCont contp, TSEvent event, void *e // session may fire interleaved in HTTP/2. Thus, in order to get // non-garbled JSON content, we accumulate the data for an entire // transaction and write that atomically once the transaction is completed. - TransactionData *txnData = new TransactionData; + TransactionData *txnData = new TransactionData(txnp, ssnData->get_http_version_in_client_stack()); TSUserArgSet(txnp, transaction_arg_index, txnData); // Get UUID char uuid[TS_CRUUID_STRING_LEN + 1]; @@ -270,18 +339,18 @@ TransactionData::global_transaction_handler(TSCont contp, TSEvent event, void *e std::string_view uuid_view{uuid, strnlen(uuid, TS_CRUUID_STRING_LEN)}; // Generate per transaction json records - txnData->txn_json += "{"; + txnData->_txn_json += "{"; // "connection-time":(number) TSHRTime start_time; TSHttpTxnMilestoneGet(txnp, TS_MILESTONE_UA_BEGIN, &start_time); - txnData->txn_json += "\"connection-time\":" + std::to_string(start_time); + txnData->_txn_json += "\"connection-time\":" + std::to_string(start_time); // "uuid":(string) // The uuid is a header field for each message in the transaction. Use the // "all" node to apply to each message. std::string_view name = "uuid"; - txnData->txn_json += R"(,"all":{"headers":{"fields":[)" + json_entry_array(name, uuid_view); - txnData->txn_json += "]}}"; + txnData->_txn_json += R"(,"all":{"headers":{"fields":[)" + json_entry_array(name, uuid_view); + txnData->_txn_json += "]}}"; break; } @@ -304,7 +373,7 @@ TransactionData::global_transaction_handler(TSCont contp, TSEvent event, void *e TSDebug(debug_tag, "Found client request"); // We don't have an accurate view of the body size until TXN_CLOSE so we hold // off on writing the content:size node until then. - txnData->txn_json += R"(,"client-request":{)" + txnData->write_message_node_no_content(buffer, hdr_loc); + txnData->write_client_request_node_no_content(buffer, hdr_loc); TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); buffer = nullptr; } @@ -317,7 +386,7 @@ TransactionData::global_transaction_handler(TSCont contp, TSEvent event, void *e TSError("[%s] No transaction data found for the header hook we registered for.", traffic_dump::debug_tag); break; } - txnData->server_protocol_description = ssnData->get_server_protocol_description(txnp); + txnData->_server_protocol_description = ssnData->get_server_protocol_description(txnp); break; } @@ -331,34 +400,33 @@ TransactionData::global_transaction_handler(TSCont contp, TSEvent event, void *e TSMBuffer buffer; TSMLoc hdr_loc; if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &buffer, &hdr_loc)) { - txnData->txn_json += txnData->write_content_node(TSHttpTxnClientReqBodyBytesGet(txnp)) + "}"; + // The node was started above in TS_EVENT_HTTP_READ_REQUEST_HDR. Here we + // just have to finish it off by writing the content node. + txnData->_txn_json += txnData->write_content_node(TSHttpTxnClientReqBodyBytesGet(txnp)) + "}"; TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); buffer = nullptr; } if (TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &buffer, &hdr_loc)) { TSDebug(debug_tag, "Found proxy request"); - txnData->txn_json += R"(,"proxy-request":{)" + txnData->server_protocol_description + "," + - txnData->write_message_node(buffer, hdr_loc, TSHttpTxnServerReqBodyBytesGet(txnp)); + txnData->write_proxy_request_node(buffer, hdr_loc); TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); buffer = nullptr; } if (TS_SUCCESS == TSHttpTxnServerRespGet(txnp, &buffer, &hdr_loc)) { TSDebug(debug_tag, "Found server response"); - txnData->txn_json += - R"(,"server-response":{)" + txnData->write_message_node(buffer, hdr_loc, TSHttpTxnServerRespBodyBytesGet(txnp)); + txnData->write_server_response_node(buffer, hdr_loc); TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); buffer = nullptr; } if (TS_SUCCESS == TSHttpTxnClientRespGet(txnp, &buffer, &hdr_loc)) { TSDebug(debug_tag, "Found proxy response"); - txnData->txn_json += - R"(,"proxy-response":{)" + txnData->write_message_node(buffer, hdr_loc, TSHttpTxnClientRespBodyBytesGet(txnp)); + txnData->write_proxy_response_node(buffer, hdr_loc); TSHandleMLocRelease(buffer, TS_NULL_MLOC, hdr_loc); buffer = nullptr; } - txnData->txn_json += "}"; - ssnData->write_transaction_to_disk(txnData->txn_json); + txnData->_txn_json += "}"; + ssnData->write_transaction_to_disk(txnData->_txn_json); delete txnData; break; } diff --git a/plugins/experimental/traffic_dump/transaction_data.h b/plugins/experimental/traffic_dump/transaction_data.h index b3a13561ae7..91046369d3d 100644 --- a/plugins/experimental/traffic_dump/transaction_data.h +++ b/plugins/experimental/traffic_dump/transaction_data.h @@ -39,11 +39,18 @@ namespace traffic_dump class TransactionData { private: + /// The TSHttpTxn of the associated HTTP transaction. + TSHttpTxn _txnp = nullptr; + + /// The HTTP version in the client-side protocol stack or empty string + /// if it was not specified there. + std::string _http_version_from_client_stack; + /** The string for the JSON content of this transaction. */ - std::string txn_json; + std::string _txn_json; /** The '"protocol" node for this transaction's server-side connection. */ - std::string server_protocol_description; + std::string _server_protocol_description; // The index to be used for the TS API for storing this TransactionData on a // per-transaction basis. @@ -65,13 +72,21 @@ class TransactionData */ static bool init(); - /// Read the txn information from TSMBuffer and write the header information. - /// This function does not write the content node. - std::string write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_loc); + /** Read the txn information from TSMBuffer and write the header information. + * This function does not write the content node. + * + * @param[in] http_version An optional specification for the HTTP "version" + * node. + */ + std::string write_message_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_loc, std::string_view http_version = ""); - /// Read the txn information from TSMBuffer and write the header information including - /// the content node describing the body characteristics. - std::string write_message_node(TSMBuffer &buffer, TSMLoc &hdr_loc, int64_t num_body_bytes); + /** Read the txn information from TSMBuffer and write the header information including + * the content node describing the body characteristics. + * + * @param[in] http_version An optional specification for the HTTP "version" + * node. + */ + std::string write_message_node(TSMBuffer &buffer, TSMLoc &hdr_loc, int64_t num_body_bytes, std::string_view http_version = ""); /// The handler callback for transaction events. static int global_transaction_handler(TSCont contp, TSEvent event, void *edata); @@ -91,6 +106,18 @@ class TransactionData */ static std::string get_sensitive_field_description(); + /** Construct a TransactionData object. + * + * Note that this constructor is private since only the global handler + * creates these at the moment. + * + * @param[in] txnp The TSHttpTxn for the associated HTTP transaction. + * + * @param[in] http_version_from_client_stack The HTTP version as specified in + * the protocol stack, or empty string if no so specified. + */ + TransactionData(TSHttpTxn txnp, std::string_view http_version_from_client_stack); + /** Inspect the field to see whether it is sensitive and return a generic value * of equal size to the original if it is. * @@ -113,6 +140,22 @@ class TransactionData * @return The view without the scheme prefix. */ std::string_view remove_scheme_prefix(std::string_view url); + + /** Write the "client-request" node to _txn_json. + * + * Note that the "content" node is not written with this function, so it will + * have to be written later. + */ + void write_client_request_node_no_content(TSMBuffer &buffer, TSMLoc &hdr_loc); + + /// Write the "proxy-request" node to _txn_json. + void write_proxy_request_node(TSMBuffer &buffer, TSMLoc &hdr_loc); + + /// Write the "server-response" node to _txn_json. + void write_server_response_node(TSMBuffer &buffer, TSMLoc &hdr_loc); + + /// Write the "proxy-response" node to _txn_json. + void write_proxy_response_node(TSMBuffer &buffer, TSMLoc &hdr_loc); }; } // namespace traffic_dump diff --git a/src/traffic_server/InkAPI.cc b/src/traffic_server/InkAPI.cc index 9dd2c3d5535..288e79e9bda 100644 --- a/src/traffic_server/InkAPI.cc +++ b/src/traffic_server/InkAPI.cc @@ -8108,6 +8108,43 @@ TSHttpTxnServerPush(TSHttpTxn txnp, const char *url, int url_len) return TS_SUCCESS; } +TSReturnCode +TSHttpTxnClientStreamIdGet(TSHttpTxn txnp, uint64_t *stream_id) +{ + sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); + sdk_assert(stream_id != nullptr); + + auto *sm = reinterpret_cast(txnp); + auto *stream = dynamic_cast(sm->ua_txn); + if (stream == nullptr) { + return TS_ERROR; + } + *stream_id = stream->get_id(); + return TS_SUCCESS; +} + +TSReturnCode +TSHttpTxnClientStreamPriorityGet(TSHttpTxn txnp, TSHttpPriority *priority) +{ + static_assert(sizeof(TSHttpPriority) >= sizeof(TSHttp2Priority), + "TSHttpPriorityType is incorrectly smaller than TSHttp2Priority."); + sdk_assert(sdk_sanity_check_txn(txnp) == TS_SUCCESS); + sdk_assert(priority != nullptr); + + auto *sm = reinterpret_cast(txnp); + auto *stream = dynamic_cast(sm->ua_txn); + if (stream == nullptr) { + return TS_ERROR; + } + + auto *priority_out = reinterpret_cast(priority); + priority_out->priority_type = HTTP_PRIORITY_TYPE_HTTP_2; + priority_out->stream_dependency = stream->get_transaction_priority_dependence(); + priority_out->weight = stream->get_transaction_priority_weight(); + + return TS_SUCCESS; +} + TSReturnCode TSAIORead(int fd, off_t offset, char *buf, size_t buffSize, TSCont contp) { diff --git a/tests/gold_tests/pluginTest/traffic_dump/gold/200_http10.gold b/tests/gold_tests/pluginTest/traffic_dump/gold/200_http10.gold new file mode 100644 index 00000000000..c00bef832c3 --- /dev/null +++ b/tests/gold_tests/pluginTest/traffic_dump/gold/200_http10.gold @@ -0,0 +1,11 @@ +`` +> GET /`` HTTP/1.0 +> Host: www.notls.com`` +> User-Agent: curl/`` +> Accept: */* +`` +< HTTP/1.0 200 OK +< Content-Length: 0 +< Set-Cookie: classified_not_for_logging +< Date: `` +`` diff --git a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py index e6e95a1bfa0..9e1aa12c274 100644 --- a/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py +++ b/tests/gold_tests/pluginTest/traffic_dump/traffic_dump.test.py @@ -204,11 +204,11 @@ # Execute the second transaction. tr = Test.AddTestRun("Second transaction") tr.Processes.Default.Command = \ - ('curl http://127.0.0.1:{0}/two -H"Host: www.notls.com" ' + ('curl --http1.0 http://127.0.0.1:{0}/two -H"Host: www.notls.com" ' '-H"X-Request-2: also_very_sensitive" --verbose'.format( ts.Variables.port)) tr.Processes.Default.ReturnCode = 0 -tr.Processes.Default.Streams.stderr = "gold/200.gold" +tr.Processes.Default.Streams.stderr = "gold/200_http10.gold" tr.StillRunningAfter = server tr.StillRunningAfter = ts @@ -221,12 +221,14 @@ "--sensitive-fields x-request-1 " "--sensitive-fields x-request-2 ") tr.Setup.CopyAs(verify_replay, Test.RunDirectory) -tr.Processes.Default.Command = 'python3 {0} {1} {2} {3} --client-protocols "{4}"'.format( - verify_replay, - os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), - replay_file_session_1, - sensitive_fields_arg, - http_protocols) +tr.Processes.Default.Command = \ + ('python3 {0} {1} {2} {3} --client-http-version "1.1" ' + '--client-protocols "{4}"'.format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + replay_file_session_1, + sensitive_fields_arg, + http_protocols)) tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts @@ -234,11 +236,13 @@ # Verify the properties of the replay file for the second transaction. tr = Test.AddTestRun("Verify the json content of the second session") tr.Setup.CopyAs(verify_replay, Test.RunDirectory) -tr.Processes.Default.Command = "python3 {0} {1} {2} {3} --request-target '/two'".format( - verify_replay, - os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), - replay_file_session_2, - sensitive_fields_arg) +tr.Processes.Default.Command = \ + ('python3 {0} {1} {2} {3} --client-http-version "1.0" ' + '--request-target "/two"'.format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + replay_file_session_2, + sensitive_fields_arg)) tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts @@ -418,12 +422,14 @@ tr = Test.AddTestRun("Verify the client protocol stack.") h2_protocols = "http,tls,tcp,ip" tr.Setup.CopyAs(verify_replay, Test.RunDirectory) -tr.Processes.Default.Command = 'python3 {0} {1} {2} --client-protocols "{3}" --client-tls-features "{4}"'.format( - verify_replay, - os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), - replay_file_session_9, - h2_protocols, - client_tls_features) +tr.Processes.Default.Command = \ + ('python3 {0} {1} {2} --client-http-version "2" ' + '--client-protocols "{3}" --client-tls-features "{4}"'.format( + verify_replay, + os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), + replay_file_session_9, + h2_protocols, + client_tls_features)) tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = server tr.StillRunningAfter = ts @@ -457,7 +463,7 @@ tr = Test.AddTestRun("Verify the client protocol stack.") tr.Setup.CopyAs(verify_replay, Test.RunDirectory) -tr.Processes.Default.Command = 'python3 {0} {1} {2} --client-protocols "{3}"'.format( +tr.Processes.Default.Command = 'python3 {0} {1} {2} --client-http-version "1.1" --client-protocols "{3}"'.format( verify_replay, os.path.join(Test.Variables.AtsTestToolsDir, 'lib', 'replay_schema.json'), replay_file_session_10, diff --git a/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py b/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py index a741e810f37..7a4dbe18dcb 100644 --- a/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py +++ b/tests/gold_tests/pluginTest/traffic_dump/verify_replay.py @@ -254,6 +254,20 @@ def verify_server_tls_features(replay_json, expected_tls_features): return True +def verify_client_http_version(replay_json, expected_client_http_version): + try: + found_version = replay_json['sessions'][0]['transactions'][0]['client-request']['version'] + except KeyError: + print("Could not find client-request:version node in the replay file.") + return False + + if expected_client_http_version != found_version: + print('Expected client version of "{}", but found "{}"'.format( + expected_client_http_version, found_version)) + return False + return True + + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("schema_file", @@ -278,6 +292,8 @@ def parse_args(): help="The TLS values to expect for the client connection.") parser.add_argument("--server-tls-features", help="The TLS values to expect for the server connection.") + parser.add_argument("--client-http-version", + help="The client HTTP version to expect") return parser.parse_args() @@ -323,6 +339,9 @@ def main(): if args.server_tls_features and not verify_server_tls_features(replay_json, args.server_tls_features): return 1 + if args.client_http_version and not verify_client_http_version(replay_json, args.client_http_version): + return 1 + return 0 diff --git a/tests/tools/lib/replay_schema.json b/tests/tools/lib/replay_schema.json index 07708bde081..3f9f470ca27 100644 --- a/tests/tools/lib/replay_schema.json +++ b/tests/tools/lib/replay_schema.json @@ -188,7 +188,33 @@ "version": { "description": "HTTP version", "type": "string", - "enum": ["0.9", "1.0", "1.1", "2.0"] + "enum": ["0.9", "1.0", "1.1", "2", "3"] + }, + "http2": { + "description": "A description of the HTTP/2 properties", + "type": "object", + "required": [ "stream-id" ], + "properties": { + "stream-id": { + "description": "The HTTP/2 stream identifier.", + "type": "integer" + }, + "priority": { + "description": "A description of the HTTP/2 properties", + "type": "object", + "required": [ "stream-dependency", "weight" ], + "properties": { + "stream-dependency": { + "description": "The stream this stream depends upon.", + "type": "integer" + }, + "weight": { + "description": "The priority weight assigned to the stream dependency.", + "type": "integer" + } + } + } + } }, "scheme": { "description": "HTTP scheme (request).", @@ -225,7 +251,7 @@ "version": { "description": "HTTP version", "type": "string", - "enum": ["0.9", "1.0", "1.1", "2.0"] + "enum": ["0.9", "1.0", "1.1", "2", "3"] }, "status": { "description": "Status code.",