Make failure reasons to be flags.

configs/access_log_format_helper.template.json

@@ -1,15 +1,15 @@
 {% macro ingress_sampled_log() %}
-  "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n"
+  "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n"
 {% endmacro %}
 
 {% macro ingress_full() %}
-  "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n"
+  "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\"\n"
 {% endmacro %}
 
 {% macro egress_error_log() %}
-  "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"\n"
+  "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"\n"
 {% endmacro %}
 
 {% macro egress_error_amazon_service() %}
-  "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" \"%RESP(X-AMZN-RequestId)%\"\n"
+  "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" \"%RESP(X-AMZN-RequestId)%\"\n"
 {% endmacro %}

docs/configuration/http_conn_man/access_log.rst

@@ -71,8 +71,8 @@ The following command operators are supported:
 %DURATION%
   Total duration in milliseconds of the request from the start time to the last byte out.
 
-%FAILURE_REASON%
-  Additional failure reason if any in addition to response code. Possible values are:
+%RESPONSE_FLAGS%
+  Additional details about the response, if any. Possible values are:
 
   * **LH**: Local service failed :ref:`health check request <arch_overview_health_checking>` in addition to 503 response code.
   * **UH**: No healthy upstream hosts in upstream cluster in addition to 503 response code.
@@ -83,7 +83,7 @@ The following command operators are supported:
   * **UC**: Upstream connection termination in addition to 503 response code.
   * **UO**: Upstream overflow (:ref:`circuit breaking <arch_overview_circuit_break>`) in addition to 503 response code.
   * **NR**: No :ref:`route configured <arch_overview_http_routing>` for a given request in addition to 404 response code.
-  * **FI**: The request was aborted as a result of :ref:`fault injection <config_http_filters_fault_injection>`.
+  * **FI**: The request was aborted with an injected response code via :ref:`fault injection <config_http_filters_fault_injection>`.
 
 %UPSTREAM_HOST%
   Upstream host URL (e.g., tcp://ip:port for TCP connections).
@@ -107,7 +107,7 @@ If custom format is not specified, Envoy uses the following default format:
 .. code-block:: none
 
   [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%"
-  %RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION%
+  %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION%
   %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%"
   "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n
 

examples/grpc-bridge/config/s2s-python-envoy.json

@@ -13,7 +13,7 @@
             "access_log": [
               {
                 "path": "/var/log/envoy/egress_http.log",
-                "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"\n"
+                "format": "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH):256% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"\n"
               }
             ],
             "stat_prefix": "egress_http",

include/envoy/http/access_log.h

@@ -10,29 +10,27 @@
 namespace Http {
 namespace AccessLog {
 
-enum class FailureReason {
-  // No failure.
-  None,
+enum ResponseFlag {
   // Local server healthcheck failed.
-  FailedLocalHealthCheck,
+  FailedLocalHealthCheck = 0x1,
   // No healthy upstream.
-  NoHealthyUpstream,
+  NoHealthyUpstream = 0x2,
   // Request timeout on upstream.
-  UpstreamRequestTimeout,
+  UpstreamRequestTimeout = 0x4,
   // Local codec level reset was sent on the stream.
-  LocalReset,
+  LocalReset = 0x8,
   // Remote codec level reset was received on the stream.
-  UpstreamRemoteReset,
+  UpstreamRemoteReset = 0x10,
   // Local reset by a connection pool due to an initial connection failure.
-  UpstreamConnectionFailure,
+  UpstreamConnectionFailure = 0x20,
   // If the stream was locally reset due to connection termination.
-  UpstreamConnectionTermination,
+  UpstreamConnectionTermination = 0x40,
   // The stream was reset because of a resource overflow.
-  UpstreamOverflow,
+  UpstreamOverflow = 0x80,
   // No route found for a given request.
-  NoRouteFound,
-  // Used when faults (abort with error codes) are injected.
-  FaultInjected,
+  NoRouteFound = 0x100,
+  // Abort with error code is injected.
+  FaultInjected = 0x200
 };
 
 /**
@@ -43,13 +41,12 @@ class RequestInfo {
   virtual ~RequestInfo() {}
 
   /**
-   * filter can trigger this callback on failed response to provide more details about
-   * failure.
+   * Each filter can set independent response flag, flags are accumulated.
    */
-  virtual void onFailedResponse(FailureReason failure_reason) PURE;
+  virtual void setResponseFlag(ResponseFlag response_flag) PURE;
 
   /**
-   * filter can trigger this callback when an upstream host has been selected.
+   * Filter can trigger this callback when an upstream host has been selected.
    */
   virtual void onUpstreamHostSelected(Upstream::HostDescriptionPtr host) PURE;
 
@@ -89,9 +86,9 @@ class RequestInfo {
   virtual std::chrono::milliseconds duration() const PURE;
 
   /**
-   * @return the failure reason for richer log experience.
+   * @return whether response flag is set or not.
    */
-  virtual FailureReason failureReason() const PURE;
+  virtual bool getResponseFlag(ResponseFlag response_flag) const PURE;
 
   /**
    * @return upstream host description.

source/common/http/access_log/access_log_formatter.cc

@@ -6,50 +6,86 @@
 namespace Http {
 namespace AccessLog {
 
-const std::string FilterReasonUtils::NONE = "-";
-const std::string FilterReasonUtils::FAILED_LOCAL_HEALTH_CHECK = "LH";
-const std::string FilterReasonUtils::NO_HEALTHY_UPSTREAM = "UH";
-const std::string FilterReasonUtils::UPSTREAM_REQUEST_TIMEOUT = "UT";
-const std::string FilterReasonUtils::LOCAL_RESET = "LR";
-const std::string FilterReasonUtils::UPSTREAM_REMOTE_RESET = "UR";
-const std::string FilterReasonUtils::UPSTREAM_CONNECTION_FAILURE = "UF";
-const std::string FilterReasonUtils::UPSTREAM_CONNECTION_TERMINATION = "UC";
-const std::string FilterReasonUtils::UPSTREAM_OVERFLOW = "UO";
-const std::string FilterReasonUtils::NO_ROUTE_FOUND = "NR";
-const std::string FilterReasonUtils::FAULT_INJECTED = "FI";
-
-const std::string& FilterReasonUtils::toShortString(FailureReason failure_reason) {
-  switch (failure_reason) {
-  case FailureReason::None:
-    return NONE;
-  case FailureReason::FailedLocalHealthCheck:
-    return FAILED_LOCAL_HEALTH_CHECK;
-  case FailureReason::NoHealthyUpstream:
-    return NO_HEALTHY_UPSTREAM;
-  case FailureReason::UpstreamRequestTimeout:
-    return UPSTREAM_REQUEST_TIMEOUT;
-  case FailureReason::LocalReset:
-    return LOCAL_RESET;
-  case FailureReason::UpstreamRemoteReset:
-    return UPSTREAM_REMOTE_RESET;
-  case FailureReason::UpstreamConnectionFailure:
-    return UPSTREAM_CONNECTION_FAILURE;
-  case FailureReason::UpstreamConnectionTermination:
-    return UPSTREAM_CONNECTION_TERMINATION;
-  case FailureReason::UpstreamOverflow:
-    return UPSTREAM_OVERFLOW;
-  case FailureReason::NoRouteFound:
-    return NO_ROUTE_FOUND;
-  case FailureReason::FaultInjected:
-    return FAULT_INJECTED;
+const std::string ResponseFlagUtils::NONE = "-";
+const std::string ResponseFlagUtils::FAILED_LOCAL_HEALTH_CHECK = "LH";
+const std::string ResponseFlagUtils::NO_HEALTHY_UPSTREAM = "UH";
+const std::string ResponseFlagUtils::UPSTREAM_REQUEST_TIMEOUT = "UT";
+const std::string ResponseFlagUtils::LOCAL_RESET = "LR";
+const std::string ResponseFlagUtils::UPSTREAM_REMOTE_RESET = "UR";
+const std::string ResponseFlagUtils::UPSTREAM_CONNECTION_FAILURE = "UF";
+const std::string ResponseFlagUtils::UPSTREAM_CONNECTION_TERMINATION = "UC";
+const std::string ResponseFlagUtils::UPSTREAM_OVERFLOW = "UO";
+const std::string ResponseFlagUtils::NO_ROUTE_FOUND = "NR";
+const std::string ResponseFlagUtils::FAULT_INJECTED = "FI";
+
+void ResponseFlagUtils::appendString(std::string& result, const std::string& append) {
+  if (result.empty()) {
+    result = append;
+  } else {
+    result += "," + append;
+  }
+}
+
+const std::string ResponseFlagUtils::toShortString(const RequestInfo& request_info) {
+  std::string result;
+
+  if (request_info.getResponseFlag(ResponseFlag::FailedLocalHealthCheck)) {
+    appendString(result, FAILED_LOCAL_HEALTH_CHECK);
+  }
+
+  if (request_info.getResponseFlag(ResponseFlag::NoHealthyUpstream)) {
+    appendString(result, NO_HEALTHY_UPSTREAM);
+  }
+
+  if (request_info.getResponseFlag(ResponseFlag::UpstreamRequestTimeout)) {
+    appendString(result, UPSTREAM_REQUEST_TIMEOUT);
+  }
+
+  if (request_info.getResponseFlag(ResponseFlag::LocalReset)) {
+    appendString(result, LOCAL_RESET);
+  }
+
+  if (request_info.getResponseFlag(ResponseFlag::UpstreamRemoteReset)) {
+    appendString(result, UPSTREAM_REMOTE_RESET);
   }
 
-  throw std::invalid_argument("Unknown failure_reason");
+  if (request_info.getResponseFlag(ResponseFlag::UpstreamConnectionFailure)) {
+    appendString(result, UPSTREAM_CONNECTION_FAILURE);
+  }
+
+  if (request_info.getResponseFlag(ResponseFlag::UpstreamConnectionTermination)) {
+    appendString(result, UPSTREAM_CONNECTION_TERMINATION);
+  }
+
+  if (request_info.getResponseFlag(ResponseFlag::UpstreamOverflow)) {
+    appendString(result, UPSTREAM_OVERFLOW);
+  }
+
+  if (request_info.getResponseFlag(ResponseFlag::NoRouteFound)) {
+    appendString(result, NO_ROUTE_FOUND);
+  }
+
+  if (request_info.getResponseFlag(ResponseFlag::FaultInjected)) {
+    appendString(result, FAULT_INJECTED);
+  }
+
+  return result.empty() ? NONE : result;
+}
+
+bool ResponseFlagUtils::isTraceableFailure(const Http::AccessLog::RequestInfo& request_info) {
+  return request_info.getResponseFlag(Http::AccessLog::ResponseFlag::NoHealthyUpstream) |
+         request_info.getResponseFlag(Http::AccessLog::ResponseFlag::UpstreamConnectionFailure) |
+         request_info.getResponseFlag(Http::AccessLog::ResponseFlag::UpstreamOverflow) |
+         request_info.getResponseFlag(Http::AccessLog::ResponseFlag::UpstreamRequestTimeout) |
+         request_info.getResponseFlag(
+             Http::AccessLog::ResponseFlag::UpstreamConnectionTermination) |
+         request_info.getResponseFlag(Http::AccessLog::ResponseFlag::NoRouteFound) |
+         request_info.getResponseFlag(Http::AccessLog::ResponseFlag::FaultInjected);
 }
 
 const std::string AccessLogFormatUtils::DEFAULT_FORMAT =
     "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" "
-    "%RESPONSE_CODE% %FAILURE_REASON% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "
+    "%RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "
     "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "
     "\"%REQ(X-FORWARDED-FOR)%\" \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" "
     "\"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\"\n";
@@ -210,9 +246,9 @@ RequestInfoFormatter::RequestInfoFormatter(const std::string& field_name) {
     field_extractor_ = [](const RequestInfo& request_info) {
       return std::to_string(request_info.duration().count());
     };
-  } else if (field_name == "FAILURE_REASON") {
+  } else if (field_name == "RESPONSE_FLAGS") {
     field_extractor_ = [](const RequestInfo& request_info) {
-      return FilterReasonUtils::toShortString(request_info.failureReason());
+      return ResponseFlagUtils::toShortString(request_info);
     };
   } else if (field_name == "UPSTREAM_HOST") {
     field_extractor_ = [](const RequestInfo& request_info) {

source/common/http/access_log/access_log_formatter.h

@@ -6,14 +6,16 @@ namespace Http {
 namespace AccessLog {
 
 /**
- * Util class for FilterReason.
+ * Util class for ResponseFlags.
  */
-class FilterReasonUtils {
+class ResponseFlagUtils {
 public:
-  static const std::string& toShortString(FailureReason failure_reason);
+  static const std::string toShortString(const RequestInfo& request_info);
+  static bool isTraceableFailure(const Http::AccessLog::RequestInfo& request_info);
 
 private:
-  FilterReasonUtils(){};
+  ResponseFlagUtils();
+  static void appendString(std::string& result, const std::string& append);
 
   const static std::string NONE;
   const static std::string FAILED_LOCAL_HEALTH_CHECK;

source/common/http/access_log/request_info_impl.h

@@ -26,11 +26,11 @@ struct RequestInfoImpl : public RequestInfo {
                                                                  start_time_);
   }
 
-  void onFailedResponse(Http::AccessLog::FailureReason failure_reason) override {
-    failure_reason_ = failure_reason;
+  void setResponseFlag(Http::AccessLog::ResponseFlag response_flag) override {
+    response_flags_ |= response_flag;
   }
 
-  Http::AccessLog::FailureReason failureReason() const override { return failure_reason_; }
+  bool getResponseFlag(ResponseFlag flag) const override { return response_flags_ & flag; }
 
   void onUpstreamHostSelected(Upstream::HostDescriptionPtr host) override { upstream_host_ = host; }
 
@@ -45,7 +45,7 @@ struct RequestInfoImpl : public RequestInfo {
   uint64_t bytes_received_{};
   Optional<uint32_t> response_code_;
   uint64_t bytes_sent_{};
-  FailureReason failure_reason_{FailureReason::None};
+  uint64_t response_flags_{};
   Upstream::HostDescriptionPtr upstream_host_{};
   bool hc_request_{};
 };

source/common/http/conn_manager_utility.cc

@@ -1,5 +1,6 @@
 #include "conn_manager_utility.h"
 
+#include "common/http/access_log/access_log_formatter.h"
 #include "common/http/headers.h"
 #include "common/http/utility.h"
 #include "common/network/utility.h"
@@ -148,7 +149,7 @@ bool ConnectionManagerUtility::shouldTraceRequest(
   case Http::TracingType::All:
     return true;
   case Http::TracingType::UpstreamFailure:
-    return request_info.failureReason() != Http::AccessLog::FailureReason::None;
+    return Http::AccessLog::ResponseFlagUtils::isTraceableFailure(request_info);
   }
 
   // Compiler enforces switch above to cover all the cases and it's impossible to be here,

source/common/http/conn_manager_utility.h

@@ -27,6 +27,8 @@ class ConnectionManagerUtility {
                                  const Optional<TracingConnectionManagerConfig>& config);
 
 private:
+  static bool isTraceableFailure(const Http::AccessLog::RequestInfo& request_info);
+
   // NOTE: This is used for stable randomness in the case where the route table does not use any
   //       runtime rules. If runtime rules are used, we use true randomness which is slower but
   //       provides behavior that most consumers would expect.

source/common/http/filter/fault_filter.cc

@@ -137,7 +137,7 @@ void FaultFilter::abortWithHTTPStatus() {
                                   "fault.http.abort.http_status", config_->abortCode()))}}};
   callbacks_->encodeHeaders(std::move(response_headers), true);
   config_->stats().aborts_injected_.inc();
-  callbacks_->requestInfo().onFailedResponse(Http::AccessLog::FailureReason::FaultInjected);
+  callbacks_->requestInfo().setResponseFlag(Http::AccessLog::ResponseFlag::FaultInjected);
 }
 
 void FaultFilter::resetTimerState() {