TS-4115: add multi origin feature to parent selection.

doc/admin-guide/files/parent.config.en.rst

@@ -138,6 +138,18 @@ The following list shows the possible actions and their allowed values.
     list, then the request will be re-tried from a server found in this list
     using a consistent hash of the url.
 
+.. _parent-config-format-parent-is-proxy:
+
+``parent_is_proxy``
+    One of the following values:
+
+    -  ``true`` - This is the default.  The list of parents and secondary parents
+        are proxy cache servers.
+    -  ``false`` - The list of parents and secondary parents are the origin
+        servers ``go_direct`` flag is ignored and origins are selected using
+        the specified ``round_robin`` algorithm.  The FQDN is removed from
+        the http request line.
+
 .. _parent-config-format-round-robin:
 
 ``round_robin``

doc/admin-guide/files/records.config.en.rst

@@ -1092,6 +1092,33 @@ Parent Proxy Configuration
 
    Don't try to resolve DNS, forward all DNS requests to the parent. This is off (``0``) by default.
 
+.. ts:cv:: CONFIG proxy.config.http.parent_origin.retry_enabled INT 0
+   :reloadable:
+   :overridable:
+
+   Enable simple and or dead server retry.  Defaults to 0, (disabled).  To enable simple retry, set
+   to 1.  To enable dead server retry, set to 2.  To enable both simple and dead server retry, set
+   to 3.  This setting is overridable using header_rewrite.
+
+   simple retry is used to retry a request using a new parent origin server, see `parent.config`, if 
+   a 404 response is received on the first attempt.  This is useful in the case where IP video is being
+   assembled for delivery accross multiple load balanced origin servers and the file may not yet be
+   available on the first parent selected for the request.
+
+   dead server retry is used to retry a request using a new parent if a parent origin server, see 
+   `parent.config`, responds with a 503.  The parent that returns the 503 will be marked down for
+   a period of time specified in the ``proxy.config.http.parent_proxy.retry_time`` setting.  By
+   default, a 503 response will trigger a dead server retry but, a list of response codes may be
+   specified in the ``proxy.config.http.parent_orign.dead_server_retry_response_codes`` setting.
+
+.. ts:cv:: CONFIG proxy.config.http.parent_origin.dead_server_retry_response_codes STRING 503
+   :reloadable:
+   :overridable:
+
+   A comma separated list of response codes that will initiate a transaction retry using another parent
+   origin server when ``proxy.config.http.parent_origin.dead_server_retry_enabled`` is enabled.  Defaults to
+   503.  This setting is overridable using header_rewrite.
+
 HTTP Connection Timeouts
 ========================
 

doc/developer-guide/api/functions/TSHttpOverridableConfig.en.rst

@@ -142,6 +142,8 @@ The following configurations (from ``records.config``) are overridable.
 |   :ts:cv:`proxy.config.http.cache.range.write`
 |   :ts:cv:`proxy.config.http.global_user_agent_header`
 |   :ts:cv:`proxy.config.http.slow.log.threshold`
+|   :ts:cv:`proxy.config.http.parent_origin.retry_enabled`
+|   :ts:cv:`proxy.config.http.parent_origin.dead_server_retry_response_codes`
 
 Examples
 ========

lib/ts/apidefs.h.in

@@ -693,6 +693,8 @@ typedef enum {
   TS_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS,
   TS_CONFIG_HTTP_CACHE_MAX_OPEN_WRITE_RETRIES,
   TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY,
+  TS_CONFIG_HTTP_PARENT_ORIGIN_RETRY_ENABLED,
+  TS_CONFIG_HTTP_DEAD_SERVER_RETRY_RESPONSE_CODES,
   TS_CONFIG_LAST_ENTRY
 } TSOverridableConfigKey;
 

mgmt/RecordsConfig.cc

@@ -519,6 +519,14 @@ static const RecordElement RecordsConfig[] =
   {RECT_CONFIG, "proxy.config.http.forward.proxy_auth_to_parent", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, NULL, RECA_NULL}
   ,
 
+  //        ###################################
+  //        # parent origin configuration     #
+  //        ###################################
+  {RECT_CONFIG, "proxy.config.http.parent_origin.retry_enabled", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_INT, "[0-3]", RECA_NULL}
+  ,
+  {RECT_CONFIG, "proxy.config.http.parent_origin.dead_server_retry_response_codes", RECD_STRING, "503", RECU_DYNAMIC, RR_NULL, RECC_STR, "^([0-9]+,)$", RECA_NULL}
+  ,
+
   //        ###################################
   //        # NO DNS DOC IN CACHE             #
   //        ###################################

plugins/experimental/ts_lua/ts_lua_http_config.c

@@ -113,6 +113,8 @@ typedef enum {
   TS_LUA_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS = TS_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS,
   TS_LUA_CONFIG_HTTP_CACHE_MAX_OPEN_WRITE_RETRIES = TS_CONFIG_HTTP_CACHE_MAX_OPEN_WRITE_RETRIES,
   TS_LUA_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY = TS_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY,
+  TS_LUA_CONFIG_HTTP_PARENT_ORIGIN_RETRY_ENABLED = TS_CONFIG_HTTP_PARENT_ORIGIN_RETRY_ENABLED,
+  TS_LUA_CONFIG_HTTP_DEAD_SERVER_RETRY_RESPONSE_CODES = TS_CONFIG_HTTP_DEAD_SERVER_RETRY_RESPONSE_CODES,
   TS_LUA_CONFIG_LAST_ENTRY = TS_CONFIG_LAST_ENTRY,
 } TSLuaOverridableConfigKey;
 
@@ -196,7 +198,9 @@ ts_lua_var_item ts_lua_http_config_vars[] = {
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_OPEN_WRITE_FAIL_ACTION),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_ENABLE_REDIRECTION), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_NUMBER_OF_REDIRECTIONS),
   TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_CACHE_MAX_OPEN_WRITE_RETRIES),
-  TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_LAST_ENTRY),
+  TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_REDIRECT_USE_ORIG_CACHE_KEY),
+  TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_PARENT_ORIGIN_RETRY_ENABLED),
+  TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_HTTP_DEAD_SERVER_RETRY_RESPONSE_CODES), TS_LUA_MAKE_VAR_ITEM(TS_LUA_CONFIG_LAST_ENTRY),
 };
 
 // Needed to make sure we have the latest list of overridable http config vars when compiling

proxy/InkAPI.cc

@@ -7980,6 +7980,15 @@ _conf_to_memberp(TSOverridableConfigKey conf, OverridableHttpConfigParams *overr
     typ = OVERRIDABLE_TYPE_INT;
     ret = &overridableHttpConfig->redirect_use_orig_cache_key;
     break;
+  case TS_CONFIG_HTTP_PARENT_ORIGIN_RETRY_ENABLED:
+    typ = OVERRIDABLE_TYPE_INT;
+    ret = &overridableHttpConfig->parent_origin_retry_enabled;
+    break;
+  case TS_CONFIG_HTTP_DEAD_SERVER_RETRY_RESPONSE_CODES:
+    ret = &overridableHttpConfig->dead_server_retry_response_codes;
+    typ = OVERRIDABLE_TYPE_STRING;
+    break;
+
   // This helps avoiding compiler warnings, yet detect unhandled enum members.
   case TS_CONFIG_NULL:
   case TS_CONFIG_LAST_ENTRY:
@@ -8134,6 +8143,13 @@ TSHttpTxnConfigStringSet(TSHttpTxn txnp, TSOverridableConfigKey conf, const char
       s->t_state.txn_conf->body_factory_template_base_len = 0;
     }
     break;
+  case TS_CONFIG_HTTP_DEAD_SERVER_RETRY_RESPONSE_CODES:
+    if (value && length > 0) {
+      s->t_state.txn_conf->dead_server_retry_response_codes = const_cast<char *>(value); // The "core" likes non-const char*
+    } else {
+      s->t_state.txn_conf->dead_server_retry_response_codes = NULL;
+    }
+    break;
   default:
     return TS_ERROR;
     break;
@@ -8451,6 +8467,8 @@ TSHttpTxnConfigFind(const char *name, int length, TSOverridableConfigKey *conf,
     case 'd':
       if (!strncmp(name, "proxy.config.http.down_server.abort_threshold", length))
         cnf = TS_CONFIG_HTTP_DOWN_SERVER_ABORT_THRESHOLD;
+      if (!strncmp(name, "proxy.config.http.parent_origin.retry_enabled", length))
+        cnf = TS_CONFIG_HTTP_PARENT_ORIGIN_RETRY_ENABLED;
       break;
     case 'n':
       if (!strncmp(name, "proxy.config.http.cache.ignore_authentication", length))
@@ -8617,6 +8635,12 @@ TSHttpTxnConfigFind(const char *name, int length, TSOverridableConfigKey *conf,
     if (!strncmp(name, "proxy.config.http.connect_attempts_max_retries_dead_server", length))
       cnf = TS_CONFIG_HTTP_CONNECT_ATTEMPTS_MAX_RETRIES_DEAD_SERVER;
     break;
+
+  case 64:
+    if (!strncmp(name, "proxy.config.http.parent_origin.dead_server_retry_response_codes", length))
+      cnf = TS_CONFIG_HTTP_DEAD_SERVER_RETRY_RESPONSE_CODES;
+    typ = TS_RECORDDATATYPE_STRING;
+    break;
   }
 
   *conf = cnf;

proxy/InkAPITest.cc

@@ -7218,7 +7218,8 @@ const char *SDK_Overridable_Configs[TS_CONFIG_LAST_ENTRY] = {
   "proxy.config.http.auth_server_session_private", "proxy.config.http.slow.log.threshold", "proxy.config.http.cache.generation",
   "proxy.config.body_factory.template_base", "proxy.config.http.cache.open_write_fail_action",
   "proxy.config.http.redirection_enabled", "proxy.config.http.number_of_redirections",
-  "proxy.config.http.cache.max_open_write_retries", "proxy.config.http.redirect_use_orig_cache_key"};
+  "proxy.config.http.cache.max_open_write_retries", "proxy.config.http.redirect_use_orig_cache_key",
+  "proxy.config.http.parent_origin.retry_enabled", "proxy.config.http.parent_origin.dead_server_retry_response_codes"};
 
 REGRESSION_TEST(SDK_API_OVERRIDABLE_CONFIGS)(RegressionTest *test, int /* atype ATS_UNUSED */, int *pstatus)
 {

proxy/ParentConsistentHash.cc

@@ -105,7 +105,7 @@ ParentConsistentHash::selectParent(const ParentSelectionPolicy *policy, bool fir
 
   // Should only get into this state if we are supposed to go direct.
   if (parents[PRIMARY] == NULL && parents[SECONDARY] == NULL) {
-    if (result->rec->go_direct == true) {
+    if (result->rec->go_direct == true && result->rec->parent_is_proxy) {
       result->r = PARENT_DIRECT;
     } else {
       result->r = PARENT_FAIL;
@@ -207,7 +207,7 @@ ParentConsistentHash::selectParent(const ParentSelectionPolicy *policy, bool fir
     ink_assert(result->port != 0);
     Debug("parent_select", "Chosen parent: %s.%d", result->hostname, result->port);
   } else {
-    if (result->rec->go_direct == true) {
+    if (result->rec->go_direct == true && result->rec->parent_is_proxy) {
       result->r = PARENT_DIRECT;
     } else {
       result->r = PARENT_FAIL;

proxy/ParentRoundRobin.cc

@@ -68,7 +68,7 @@ ParentRoundRobin::selectParent(const ParentSelectionPolicy *policy, bool first_c
       //   if we are supposed to go direct
       ink_assert(result->rec->go_direct == true);
       // Could not find a parent
-      if (result->rec->go_direct == true) {
+      if (result->rec->go_direct == true && result->rec->parent_is_proxy) {
         result->r = PARENT_DIRECT;
       } else {
         result->r = PARENT_FAIL;
@@ -111,7 +111,7 @@ ParentRoundRobin::selectParent(const ParentSelectionPolicy *policy, bool first_c
       // We've wrapped around so bypass if we can
       if (bypass_ok == true) {
         // Could not find a parent
-        if (result->rec->go_direct == true) {
+        if (result->rec->go_direct == true && result->rec->parent_is_proxy) {
           result->r = PARENT_DIRECT;
         } else {
           result->r = PARENT_FAIL;
@@ -166,7 +166,7 @@ ParentRoundRobin::selectParent(const ParentSelectionPolicy *policy, bool first_c
     cur_index = (cur_index + 1) % result->rec->num_parents;
   } while ((unsigned int)cur_index != result->start_parent);
 
-  if (result->rec->go_direct == true) {
+  if (result->rec->go_direct == true && result->rec->parent_is_proxy) {
     result->r = PARENT_DIRECT;
   } else {
     result->r = PARENT_FAIL;

proxy/ParentSelection.cc

@@ -478,6 +478,7 @@ ParentRecord::DefaultInit(char *val)
   this->go_direct = true;
   this->ignore_query = false;
   this->scheme = NULL;
+  this->parent_is_proxy = true;
   errPtr = ProcessParents(val, true);
 
   if (errPtr != NULL) {
@@ -560,6 +561,13 @@ ParentRecord::Init(matcher_line *line_info)
         this->ignore_query = false;
       }
       used = true;
+    } else if (strcasecmp(label, "parent_is_proxy") == 0) {
+      if (strcasecmp(val, "false") == 0) {
+        parent_is_proxy = false;
+      } else {
+        parent_is_proxy = true;
+      }
+      used = true;
     }
     // Report errors generated by ProcessParents();
     if (errPtr != NULL) {
@@ -647,6 +655,7 @@ ParentRecord::Print()
     printf(" %s:%d ", parents[i].hostname, parents[i].port);
   }
   printf(" direct=%s\n", (go_direct == true) ? "true" : "false");
+  printf(" parent_is_proxy=%s\n", ((parent_is_proxy == true) ? "true" : "false"));
 }
 
 // ParentRecord* createDefaultParent(char* val)

proxy/ParentSelection.h

@@ -90,7 +90,7 @@ class ParentRecord : public ControlBase
 public:
   ParentRecord()
     : parents(NULL), secondary_parents(NULL), num_parents(0), num_secondary_parents(0), ignore_query(false), rr_next(0),
-      go_direct(true), selection_strategy(NULL)
+      go_direct(true), parent_is_proxy(true), selection_strategy(NULL)
   {
   }
 
@@ -117,6 +117,7 @@ class ParentRecord : public ControlBase
   bool ignore_query;
   volatile uint32_t rr_next;
   bool go_direct;
+  bool parent_is_proxy;
   ParentSelectionStrategy *selection_strategy;
 };
 
@@ -163,13 +164,13 @@ struct ParentSelectionPolicy {
 class ParentSelectionStrategy
 {
 public:
-  // void selectParent(const ParentSelectionPolicy, *policy, bool firstCall, ParentResult *result, RequestData *rdata)
+  // void selectParent(const ParentSelectionPolicy *policy, bool firstCall, ParentResult *result, RequestData *rdata)
   //
   // The implementation parent lookup.
   //
   virtual void selectParent(const ParentSelectionPolicy *policy, bool firstCall, ParentResult *result, RequestData *rdata) = 0;
 
-  // void markParentDown(const ParentSelectionPolicy, *policy, ParentResult* rsult)
+  // void markParentDown(const ParentSelectionPolicy *policy, ParentResult* rsult)
   //
   //    Marks the parent pointed to by result as down
   //

proxy/http/HttpConfig.cc

@@ -1117,6 +1117,12 @@ HttpConfig::startup()
   // Local Manager
   HttpEstablishStaticConfigLongLong(c.synthetic_port, "proxy.config.admin.synthetic_port");
 
+
+  // origin parent selection variables.
+  HttpEstablishStaticConfigByte(c.oride.parent_origin_retry_enabled, "proxy.config.http.parent_origin.retry_enabled");
+  HttpEstablishStaticConfigStringAlloc(c.dead_server_retry_response_codes,
+                                       "proxy.config.http.parent_origin.dead_server_retry_response_codes");
+
   // Cluster time delta gets it own callback since it needs
   //  to use ink_atomic_swap
   c.cluster_time_delta = 0;
@@ -1378,6 +1384,10 @@ HttpConfig::reconfigure()
   // Local Manager
   params->synthetic_port = m_master.synthetic_port;
 
+  // origin parent selection variables.
+  params->oride.parent_origin_retry_enabled = m_master.oride.parent_origin_retry_enabled;
+  params->oride.dead_server_retry_response_codes = m_master.dead_server_retry_response_codes;
+
   m_id = configProcessor.set(m_id, params);
 
 #undef INT_TO_BOOL

proxy/http/HttpConfig.h

@@ -384,12 +384,12 @@ struct OverridableHttpConfigParams {
       freshness_fuzz_min_time(0), max_cache_open_read_retries(-1), cache_open_read_retry_time(10), cache_generation_number(-1),
       max_cache_open_write_retries(1), background_fill_active_timeout(60), http_chunking_size(4096), flow_high_water_mark(0),
       flow_low_water_mark(0), default_buffer_size_index(8), default_buffer_water_mark(32768), slow_log_threshold(0),
-
+      parent_origin_retry_enabled(0),
       // Strings / floats must come last
       body_factory_template_base(NULL), body_factory_template_base_len(0), proxy_response_server_string(NULL),
       proxy_response_server_string_len(0), global_user_agent_header(NULL), global_user_agent_header_size(0),
       cache_heuristic_lm_factor(0.10), freshness_fuzz_prob(0.005), background_fill_threshold(0.5), cache_open_write_fail_action(0),
-      redirection_enabled(0), redirect_use_orig_cache_key(0), number_of_redirections(1)
+      redirection_enabled(0), redirect_use_orig_cache_key(0), number_of_redirections(1), dead_server_retry_response_codes(NULL)
   {
   }
 
@@ -561,6 +561,10 @@ struct OverridableHttpConfigParams {
   MgmtInt default_buffer_size_index;
   MgmtInt default_buffer_water_mark;
   MgmtInt slow_log_threshold;
+
+  // origin parent selection simple and dead server retry.
+  MgmtByte parent_origin_retry_enabled;
+
   // IMPORTANT: Here comes all strings / floats configs.
 
   ///////////////////////////////////////////////////////////////////
@@ -595,6 +599,9 @@ struct OverridableHttpConfigParams {
   MgmtByte redirection_enabled;
   MgmtByte redirect_use_orig_cache_key;
   MgmtInt number_of_redirections;
+
+  // origin parent dead server retry response codes.
+  char *dead_server_retry_response_codes;
 };
 
 
@@ -760,6 +767,11 @@ struct HttpConfigParams : public ConfigInfo {
   ////////////////////
   MgmtInt synthetic_port;
 
+  /////////////////////////////////////////////////////
+  // parent selection origin server response codes. //
+  /////////////////////////////////////////////////////
+  char *dead_server_retry_response_codes;
+
 private:
   /////////////////////////////////////
   // operator = and copy constructor //
@@ -860,7 +872,7 @@ inline HttpConfigParams::HttpConfigParams()
     redirection_host_no_port(1), post_copy_size(2048), ignore_accept_mismatch(0), ignore_accept_language_mismatch(0),
     ignore_accept_encoding_mismatch(0), ignore_accept_charset_mismatch(0), send_100_continue_response(0),
     disallow_post_100_continue(0), parser_allow_non_http(1), max_post_size(0),
-    server_session_sharing_pool(TS_SERVER_SESSION_SHARING_POOL_THREAD), synthetic_port(0)
+    server_session_sharing_pool(TS_SERVER_SESSION_SHARING_POOL_THREAD), synthetic_port(0), dead_server_retry_response_codes(NULL)
 {
 }
 
@@ -880,6 +892,7 @@ inline HttpConfigParams::~HttpConfigParams()
   ats_free(connect_ports_string);
   ats_free(reverse_proxy_no_host_redirect);
   ats_free(url_expansions);
+  ats_free(dead_server_retry_response_codes);
 
   if (connect_ports) {
     delete connect_ports;

proxy/http/HttpDebugNames.cc

@@ -59,6 +59,8 @@ HttpDebugNames::get_server_state_name(HttpTransact::ServerState_t state)
     return "CONGEST_CONTROL_CONGESTED_ON_F";
   case HttpTransact::CONGEST_CONTROL_CONGESTED_ON_M:
     return "CONGEST_CONTROL_CONGESTED_ON_M";
+  case HttpTransact::PARENT_ORIGIN_RETRY:
+    return "PARENT_ORIGIN_RETRY";
   }
 
   return ("unknown state name");